{"_id":"@blitzjs/file-pipeline","_rev":"250-9aff3cfb7786a73d625a8c258a0713c4","name":"@blitzjs/file-pipeline","dist-tags":{"latest":"0.39.0","danger":"0.40.0-danger.3e267be38.2","canary":"0.40.0-canary.4"},"versions":{"0.13.0-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.13.0-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.13.0-canary.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6536d29b058a9b816a9c440d7ec6af3bfa2be541","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.13.0-canary.2.tgz","fileCount":26,"integrity":"sha512-otM7ONSGdlqJDACrIMnLE7dH9dVMqnaaS4rTDrkSj5AtXc/n92erja66SMwspNT/vC3yNIfwZTg+orO6WM/b4w==","signatures":[{"sig":"MEUCICxY3DianOglkMg8xJEi4B8xSWyRjY5e+SKRY06ni7LaAiEAgMXeIrRsfrTvet6YCrzLFpYnlsgZeQPf2BuRTa2HENg=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":164373,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe2mObCRA9TVsSAnZWagAAGNMP/3xUgOceCj6zacDsPh6M\nBP5vDwHEydW0xKHfJY47gDFk/frmKbE9CStUtWb3DfiT5p1qbfI+kHhhzw2M\nWAj0tPpZO0l7zqFFNZyT08e2nGUqZCrY4SBWyafxn3ztE1L4zxxbv2PluSGc\nM1sUw1T7saQGUAfQRRQFUaJHHaCY74iO505YHDPsi1s9+TJKSiyZ0K+NGQr+\nK+X2nr14mCFiF+2WNzsmfOS102MFj88dXgotpUktqTItrJ1X6wvBQpPcvmfS\nmwfyrOMkAsNW2w9znRrarNjPwZ7FomZtZlhHMEfpDdigwB/l3kzqMiQGmNTO\nV6phETbcf9MHPfeUXds4nDLs8YuJenPdVUYsU94d/hvs3S+APD3msIXt0Wut\nKUIHyrmu4zvCZ7AYjwi4VGfPegmD6hCx3l3/nukw7frafIkua1Oqyau9yi1O\n4StCfsKRVwnJ6jx8isH2pBuxg0smGSOTDIgd3iSvgPk/gGAUrIlZdyhryhEU\ndTjuc71CT9bYycREpfwXjK2M14Ccw1+GHj39Pq6wYN6BZWUh/1zyVYSNWPof\nTCQdDLflP9X+9tty7Y2RhF7s1cF57FWDL5MtLLjZMY4IaETsg+7ODMxrGxG7\nh7GKappQlwvZhH6Up23zjaVPAIfRpiG+KsLiCztZQv3nBoBn5x3wl834taZO\nIrm5\r\n=59g0\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","husky":{"hooks":{"pre-commit":"tsdx lint"}},"types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"09932642f92ed033490daf22445b8887c402940b","scripts":{"dev":"tsdx watch --verbose","lint":"tsdx lint","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"6.14.4","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.13.0-canary.2_1591370650595_0.3106196822032057","host":"s3://npm-registry-packages"}},"0.13.0-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.13.0-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.13.0-canary.3","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"762abfd5d8a0c9facd9891d76d4e795902246826","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.13.0-canary.3.tgz","fileCount":27,"integrity":"sha512-g5GlgNlOwmUiAgspH9uBhs+q7AtaN06KGgOpaxQEcliYZC6iDHs5I4E6qoKFLXfLRf00JpqTkrAPP16MzI2+TQ==","signatures":[{"sig":"MEQCIGGr7rj1w+WGyJaiHwmSLFjbi87HbI1QGuzAfF/jXD/EAiAwJ8vpACOlWv8Ykk5QmUgVIVCgrbhIjNDQGQfmvF14yg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":165443,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe2zLCCRA9TVsSAnZWagAAFxUQAIE94Py7S5wUcOLE3Am9\nvqMZPAT7bOdGCUINd4wOIajFCHyMEP/FpZRYqUA9uArxJv73f+CmKMlmdQDw\n57b83WnYpfbfGhhIfNtoo6qYwa3g8clZO/FEzwT0DVfcT4l+sc50NBu9mNVV\nAoh3xY7DbrPviAzVz5i32ugin8HFpZofhKeZ7SuhfvTAUFStXEeDfz3XpLP3\n6W/omBrNuSSSj2x+M8RZtqnOOfHaZ7N+dpKkZcuMRyRIHpG/LKFMuMcl07Ps\noiP/IwX2vzJl+PqWdvj7ridZmLAcnW/h0quhawdag79dYqe4dmUc+r5ZJ59X\nUkGaNt/nns9x8HbWOOgxMzYxGALMrip3pHxgKsgsGV4rCm8WdlhyIFICPBN1\noIrpeEQnHz0ONKmnJSHg1/vZhIzuhCMOcRowbsfkgjpOml4TB3a90D8VfGeB\nRyALwCwpAUMmU4MQQclJ8799emMTIic+w9VJwQntiaTgBNrUbhs1dP568Vw/\nsvez1kFrGocyW6kxU8zgnvtR9xf+4jNRoc+rftvfFcSY4R4ioHygZQOf0pR2\n9JHpzzTeb6cd25XQlgiJu8UkxVWxWQZz/FYHefczC+KoRgkt4B27R/vs1QWS\nWb55kJ2zZD/inNEMnndTmBN6rxZhOPf+uqDVUkaH9n7YavhRQjHPQZduxNZx\n1mWQ\r\n=QpzK\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","husky":{"hooks":{"pre-commit":"tsdx lint"}},"types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: 'This will appear in the object returned by transformation promise'}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n![](\"diagram-file-transform.png\")
\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === 'unlink'\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: '/path/to/foo',\n content: null,\n event: 'unlink',\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: '/path/to/foo',\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"15d00bc8de7390d26921336dc4a0af1ab4bd2c2b","scripts":{"dev":"tsdx watch --verbose","lint":"tsdx lint","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.13.0-canary.3_1591423682068_0.6272624946893388","host":"s3://npm-registry-packages"}},"0.13.0":{"name":"@blitzjs/file-pipeline","version":"0.13.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.13.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"ae1b47cc2b99968ae71968223dd271e05ff255cc","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.13.0.tgz","fileCount":27,"integrity":"sha512-9dIyn1Tcmr11SuA2IYWLEFoKIHiM3UflE32gf8RnISknN2OCxlEftOpU/Vwh3G6I0mEDPDK16rZ84rHNARrEMQ==","signatures":[{"sig":"MEUCIBvny3Pb30KTQ1xtAVODrRx42+iZ/eta0Q1z53c05yeXAiEAu2qsFbstK7YerJqft9zaGKnmmPsh53i/Yvso3FcsQHw=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":167169,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe22GBCRA9TVsSAnZWagAAz2QP/2Ir1lkETvj0VPkWeXNA\nw6PZ7TVHwsprGPLarPB/HTCrmpeixWuJcGFb9C4N/oCoufkxVveoMNblQzRf\ni1ua14vXKyq4Votml0gnAkwbwTJdMFQAknlhGASbRA9lbcNjJS0NkjGF0DvW\nsTSYcF8EzmpgtMlYRw70P94KlKQfxFe7m9um5ZFn5cKp7EKubZnALSJrav8T\nXycSz5jPUAN3jRICI8+cZXUhAEKKPXDTEsTNatFtdrEV+gXJDG6AVM5nYzyW\noS2HmlvHf+5zVJqstxEBc58Z8UI2VhDZH18YPCF+pZ1MYwcXFsIfDihw61Jp\nGtAeg1fHxd+q+CHSs63XQ3r/Vbaw2mZeInH6NaQkgI55xstUSl2a/TkJymSS\nxMXPPRlNQOv70PLNIWXku16jLcpG/J9qFdOww+IfNuv03Qa+xC02foKSDgL/\nitMuhI3fvbkXvwcv1Eq1xR9oKiGzLv5/hA0Szd0PovCUGShXvY2wH102Kh+v\nznlkQad33qR292YEGtxdJzVwivbNnSR3sytU6bPp3/ys3VZDKtT2awS/Xyjb\nLdbAX317I6k347CkSyI4fRMQJ3K0vgtM8HAtOmxFtqI6ITPrp0FMVaQ1DccS\nf5mB+clsWFEU5VudWA/+iHDoViaOSqm0z8jhNJebp1+E73KrBVO0gLq2Y8PL\nTY6p\r\n=uN2s\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","husky":{"hooks":{"pre-commit":"tsdx lint"}},"types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"462e22a7455a9056170b022285e97d424736f73f","scripts":{"dev":"tsdx watch --verbose","lint":"tsdx lint","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.13.0_1591435648852_0.7976127250794793","host":"s3://npm-registry-packages"}},"0.14.0-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.14.0-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.14.0-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"3028b768517393b4fd95bda2fd1407cc1ccefb17","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.14.0-canary.0.tgz","fileCount":27,"integrity":"sha512-xyUMdtTFrMEjHMxPTdfpdQfE5BPEfKYqf5jOaVucyEY383s56irxyJj3QXSg1MwtNqfK4/O36A0IQL7XrVr5Ww==","signatures":[{"sig":"MEUCIQCMmoEz96A1320ygCv/DSUcjFHeGw35x5qmqNwLTnI7egIgGMJfH/QGwVXvjXoyJA1yo5XvRFSPldt9NEYpKUmooYw=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":167811,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe5PMICRA9TVsSAnZWagAAOt4P/AyAyVmoVdcCjt+l4Lf2\nRVgOn6WvJY2pGjhiIsIt0klmheQc1z3WleK1BnCXJeBKy9JlBAwFgBmTaeWi\ncBr+UwYm9vdcw7k1V9tY/eZKVJ2NSAZ6i1deFFBwmR+owVAB/31Cw34TYqeL\nqRyhVApXV9FcEmtRf1E5mLvPCOXNQqWsbPwYl9eE3RJgXRfx8o5BwFnFi6lr\nuFkHOf4Byg8skP/tZ7LLX9c9DFvHzM9N3VD27qdGFiYDiSRocgzd2qLdDHMT\nU2n3scanNf4u6r2yb6/IxqyU1OT/BfXq2L6c/m03NDSYbDLiSimB3OY8JVH3\nudTdBhvnwKXg1g/UA2bWyQAqq6d8Zj0+8joMKY1iIX1XeesCdYyMNenTPzkO\nw74UnPrndE3VCgwdim1dcgjk+48iY1m2BEMtgWZrB4gc7aJdd9DoVsD/fhmg\nnupVXfa78L4yyrd/4TQgtjLbA81NAh80sK8PQlk1JMnJM8snlVoOxusLnRPj\n8ctClYzg7VQN96U7jXnjPZMlBajt0KUo80fsrJftTHFOaAYlBtozxEdiifWo\nWIEsiXPkZUIDaBgXWSI3VCNSQhT4sO4OeeDEzWPryCFwKyDthj5xTbEhdL4k\nCoKY74ACcZ/br+Q4bY/yjo8PzZTZNm3ikMCah9AJZ6xEvozH9q7TmMiyS9/q\nsplz\r\n=ex49\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: 'This will appear in the object returned by transformation promise'}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === 'unlink'\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: '/path/to/foo',\n content: null,\n event: 'unlink',\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: '/path/to/foo',\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"f0775cbbb9120f72f9b38870452141588da19a4b","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.14.0-canary.0_1592062727839_0.6453221451060238","host":"s3://npm-registry-packages"}},"0.14.0":{"name":"@blitzjs/file-pipeline","version":"0.14.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.14.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c66f50c7763f232876ac11f6c9e72586f2881c43","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.14.0.tgz","fileCount":27,"integrity":"sha512-nk3R/F6O3ZdgMRKYc7JvbaHxTrDo1D0pjqvSfaUKhNiodKumKn4VXivdZWOdpy+cYQqFSxclN4dWd0vIGHn56w==","signatures":[{"sig":"MEYCIQDKb6U4qq3Og0uRKiMVjpE6duFhXcqZzB9zUB1475rFOQIhAJcZ9a2koyhjbADXKkhLHeulE7cxc/Dma5U+yc+I9Vab","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":167802,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe5PhFCRA9TVsSAnZWagAA/w8P/1dLBGkMZWjy6E01T4Bk\nFwXWFo40ZZm1L1t2gsekjrvUWQVcXVF1HapWckOVZS5lQCBgfdG3xQltFZB6\nigb09IZsavaTi/vrP9Db44nkqW/M2PiEEIvheqd/KoedMHALqw3AJrK8GFnP\nG2+3XGSkCxamV8fTEvUmnpbhI02IWVwvX2Bix7PkJ9koomLc6yzXlwOX+NPA\nn06T/BCVlcxYTvtVImVaJmEjBxGSZkbmogsYkvxCNbEkgcA3pDpgg9vBWYDf\n6t0ChH9tDLklO2HghYDxxu0csLaV+N7h34jS362LFvSBMmi2Y2lOGQLm4S3t\nAK9cyInnARpY/+zIIY5n+uYQqqrBYk4y1uw9IFF3VvXMGTcHA70cKJCD6Sw1\nRVBS690wdlyMc2POCzFBFWhfZxD03xpWnVUWovuK6a476c9P7+OGLmnj0mPc\niyCmC6cQjQeiLnEkWsVb2K2sDcrZygWvvvreHygmX7BRhDZrFb3tI3ulZlKQ\nXungrdMP79xuRed2wsKsT7gis1pcO1E1hPFCLm/BuQ5Vtz0/094kqSfS3n2X\naV9yqJP7sn/JWDRmCckQNdJjV2IVfWkWq8RIzS6XAFjwEt4x8+5Q62NLSYzy\nhAvdSnP07+NVsdGUVXYkM+S6huLYcAQr/CsK/JNGSLLCGW3G77wnbQbS4K3Z\n+MVA\r\n=nDgv\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"9eb85f61ed1c50db48dad7f78ef5cdf1be916318","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.14.0_1592064069528_0.8566400229346993","host":"s3://npm-registry-packages"}},"0.14.1":{"name":"@blitzjs/file-pipeline","version":"0.14.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.14.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"97810da6099113975e79510fe3e82c973e9fd5cc","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.14.1.tgz","fileCount":27,"integrity":"sha512-H6HbzINfTEW6XJqoq76n1NMB0jteFtcZ9Owa20Wn0md8MVb4Ncvl1G7l3BZ7L5pQfp9GrdHaQgrPNbfVqEa4CA==","signatures":[{"sig":"MEQCID1qzYYge0ngahYo47Z9cVlDq+JD9LCQRXCCAoBvg9tcAiB874WHkoYFueGx5J/SUw7NB7pGVcQlta8Gs9hUy+zrXw==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":167816,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe5t6UCRA9TVsSAnZWagAAH1YP+QET3zc83NBopkHseAnG\nmQe1Wyzy2Zab7KkjFXp00Lsu7/NALmHZcoLVyNEKVAhCADcJwn4WaF/OeLKS\nS5UqOviIttvjXGKPfuXRhroV+/nP1i6kyN7AV++r7T8lYxQgdy7Zkcyf7r9H\nX8z/COxexYtqo/xZXTkWArNqLasPe6rJja+T7lwwvyoeuUWao7O0gECDJVbz\ndB38MA8qHpTBEXiKiSRMUpt5OLe8+ZD43OPcvdQ0vzoYuNc23CfK0gijI1MN\nRky+SsGxRDDgMXQ5GrqcXzpn3fIedo4RsoqCRO2bp8xlS8D8tZ7bCRicCrpI\nkJOYizUKCwsu/yzh6+p2yM20LJ6HBga6w8pLS3oyaEL2tC5Xvao+xRl2KxGT\n39QGZv+pXlQ/G/ecKns++8us7i6hpwzpZwX7jP3+buZgy4W1DIkG+Wa/cPvI\ndSbjRaAjn4FVu8/8WPvTtmWmtC4JsL4YStnWH9db40qG38OencF44a9+k338\nGZ5WfdJFRp07Eym3/N6p7c/UA8BrS7k7VltnVmTZb/L37h6B7e8oMVFrxJIh\n3dRVfy6xliwZgu2IiM2cNFdhXTHJW85RhJFsPvAP7DtdNLLh67GyosZlsvpj\nvDyAwD8xbnY5SuunIwTmnnB6udJTlddKOAq6l2UzxEFcWbHLBzzeJNFJXOjK\nv4gt\r\n=HEDV\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"278fa74e1b36ef142b312544285b791523dff005","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.14.1_1592188564092_0.3635589622262472","host":"s3://npm-registry-packages"}},"0.14.2-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.14.2-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.14.2-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"d4cdc0e0f9c300b69bf9b71e7fb1fafbe4d4acbf","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.14.2-canary.0.tgz","fileCount":27,"integrity":"sha512-0ivU1oos/xB/XPYrngVV27QC7thaPBbQd1XgZRAUKVwIVGiM6RcI3nB0ZoYtDj7HaTc18MLNpc0FCge23PXhIA==","signatures":[{"sig":"MEUCIBsNLRiQhcy7CZ2Wl8K/rw5IjenbPAz+lpe5gQwZcnw2AiEAoRGho4Tu/yPVS3/bIWMGA74lGYiwdcZIoF710gacVzA=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168088,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe5xpiCRA9TVsSAnZWagAAebYP/RLzEa4SP4gv+bEOsj7G\ntt9mm85wRqSnAhTGZ9uuBpnFF/w70YEVcBv+PihfN9WpOpgOY3ENJUSd7O/d\np2uE/z6q2EtHZN0X5oYCmS6NvZBhJzuvXD2mbsJY8A7wOnAkehGvHdqHIRu3\nT1P9Y5raRhcYxs7owLxnpnVN4TDPzIyoYrc38hz8TkA61lK8oE6Z6gDCTeyn\niRo2XN14G7LMEUMfXpbyY6KslUO/afDjbU0f9U/3JbOI9UoxE36DXuQ+NW00\nSgC8IeQZ2P/H9R3W7iPdcpchl8zKrt2vuep4Uh6ypWN0GGMmIIT9yKfrZLx7\nmy39nx4dppHs/kg7ONf1tUplK9f5pUEewV7opg9PNRI2IX2eYh2UeZwb7FW8\nkY1fi4a5p6jnXYZhsOswdhPrxMndaX+fd5H/zWTmsOa/VHkgbuVDRQ+4JUIU\na1G43vHxmfXB/UFhM/r+ZfAtpkIe+J8wW5j9PAx8lwx1Y6V14MhXpfA4V5E9\nelOSDMoeiK6vTI/PtM+7p8iTiItbMUKCfgfRCtMrtIQNgErBPgkGx8CLJ7I5\nOp3iFYtDLN8FISl0RlHyTsQdDAFICIy288YK5O+KD5Mf/4Oukx9f13MmX/+v\ndD29xKLQLSdDBlDpEdRpsflyd5Ad6G70YpLayUdYY2EtO1gCIOS0ebsuze9K\nKJBP\r\n=FK//\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: 'This will appear in the object returned by transformation promise'}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === 'unlink'\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: '/path/to/foo',\n content: null,\n event: 'unlink',\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: '/path/to/foo',\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"339ebcab7afea1324a56b3193bf105aaa1af2d40","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.0","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.14.2-canary.0_1592203873793_0.805944460321594","host":"s3://npm-registry-packages"}},"0.15.0":{"name":"@blitzjs/file-pipeline","version":"0.15.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.15.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"bc77456afa3d47911bc5ed301961038a56aacf73","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.15.0.tgz","fileCount":27,"integrity":"sha512-+eAJjbq4aS0Z7RCCnEPySrvKazyz1TBMcV4CN7Ac0xCXTXkmHARSSPXG4gQHd0F7pvp+WH0Oi2cjHvqSFZ304A==","signatures":[{"sig":"MEUCIQCkMEPpavNOz3vMD25BcDZ8vcFkCOPuTWMHZyYHXX6yYwIgYVl8spwW1dyu/H+tPOlHedCzl7o5+3BF0AMopLmNhis=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168079,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe6KPLCRA9TVsSAnZWagAAdNQP+wcn8/CoK7S11S5D321U\nZBVB8r/vbXTaZVi+hbEGcDlI9tj8nGys2SJCE2qpSQNFqwkChBi59uBMEEKm\nV0SEpXuDck1FSBZKWKAPEHwrJNi130Dv1yU6p+v790TIpfMda7qd7r3ZVG7V\nilSU3ImiiYbYDO7uLCQCaqJH078UQMdvECJRsPbeXukMqz3HGGN6KIuZ8Gum\nTQlmH7NSv8OEE8VlgeGiJIHTe5f+IlEifXxAA8uZXGbfJPpyrJJzqJI4ptBr\nU7vboi2eLiHu67niLpCWlP/AN6W3aVWQtQsZbsEqTOjrxntil2mkxTK9DIsz\nCGmRSc00DMpm1++wzlAzr35S/ltUEnW8aPlBLsekDGOY1kphSAr1dd+C0H4G\n8rSGtAsxf9BcuimwtufNAEUahWBdiQPe1SneG52itx7o86+dGK/eT4D4wWer\nLy98wnOYtVNTPIr/L5wgwvAKK14okaSSkvyVOHbD6QqX26oq0Vk4u5f8RCOh\nQrkcwuCG3fdpTHqLpnw9JWSRK67xH33hBzT1rBOfjJ64gCGYlGLlSkyXVh1E\nr7rAamZVDkwF6iBsqR3H3V/11dcnp+Thqbl9O9TeXooo7dNz+/O7w/lgSZI4\nx7yVKB0S0dIeBSjg/6Ks1lBlXOdxSLO75A57RxKKv54pXe0bLQEsa0oOIC+6\nNelo\r\n=kRQi\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"dd2d759d35139cda10665d22b2189cb6a452b23e","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.0","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.15.0_1592304587060_0.244561200476906","host":"s3://npm-registry-packages"}},"0.15.1-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.15.1-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.15.1-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"9527cec84bae493378e51a57f22be09a82600352","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.15.1-canary.0.tgz","fileCount":27,"integrity":"sha512-zasiinXU5K9KUDThalxm3VJkA8wfVK3l25pkiPRXkbdz0Sx52+IXa/A+CIAIXc6b4h7DMHKDtdJ1Bkp78jFiJw==","signatures":[{"sig":"MEUCIQD7NRNqw0Tw6rGXhqAZuQbqkNiw/alm2VsDRV5DOZcOaQIgO2cuBUfd5v+oXthDxjYC3gSrtcc3SlX5jZVTwOPj5A4=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168098,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe6aIpCRA9TVsSAnZWagAAmTYP/R3F4QIPNvZPQgS1fioX\n2YM9pEWuRoGYbcGObNpXsELFY0jTGfBIU6TdaaDxCqtv0jnq4nxknu4EFn+J\n0Q0xRDB4ESz4+NBsUX6Y0T8i9mAqxlRb4Lp+4gTrZJD3wj1mrdSKzTrdwXOc\nsoRYqwEo9j7dvYTolHvokcPimNBtkQoiEC5so1aO9Z1fJIOjjIFrYi1DFg4Z\n9XIrN8NEbRxpOjqlq9b+37ODUPeKXhsMFUM5Xz4YvUEA28iZivtLMI+i4cHw\ndczr+9gav4r23Xc6C75TJAd3KN0ExX5PNZz229NNZ6zRSfmp6se6xJ1iaAWZ\nPplvkqoaHXxNjh4nT0ckSipqY6n5qvFLpaH6NSXeInhBSKAou/+ZXNGmLqQ9\nNJa7Br4LI0WUYjNbZkE9ExzpiIU32pJ2u0YqCWNlQNDtt34d6prwpzdFUILo\n1zyKOv3XXie1bGdB6Hw1NkgYXP4CVQi+jADuKfnmQctpMuU1ZnI6VmJatiQ6\nwRamCvw2rmMGHr7NHzdn4D1tvfBl9lj5iT2kiXNerByXJUEb95zLYuhIXUbS\n+A6GP7dVBRtejyCLqohhs+Zdk9e90P16+5aP2DYRdlP2v39m121fKDk+RsbJ\nk+j9gmMRpvsvANzDetClb58Uqmt8c5pIvlkSc/AmM8OdG+D9FXDHkUVzyfk/\n1uIK\r\n=sFlU\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: 'This will appear in the object returned by transformation promise'}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === 'unlink'\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: '/path/to/foo',\n content: null,\n event: 'unlink',\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: '/path/to/foo',\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"b8e2e10288cefd637678556a80d897fd26ed74ac","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.15.1-canary.0_1592369705118_0.754799629683262","host":"s3://npm-registry-packages"}},"0.15.1":{"name":"@blitzjs/file-pipeline","version":"0.15.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.15.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"44e6d244f2450c37b9ce3ca8279d54d07a837650","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.15.1.tgz","fileCount":27,"integrity":"sha512-jvy+NP3XFs1ElxclGsU/wXsub56J4RuACZH5L7MrafpadRoo2wToe5Pvcjr+2xx4Ixs8PCC3dRGGU2ObvgNxEA==","signatures":[{"sig":"MEYCIQDDdB/GTrf97uO7DWb425Nym4ECLJyJAfPWwtRP02JvEAIhAI5FpbGBI6SWWS1vbzvwVXJHxKb6EJstePXCloS+9gza","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168089,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe6fr9CRA9TVsSAnZWagAAUJwP/jqTAPaqtmGmWSUOqigz\njzfrV9czgRtkNYSj5YnB5wywOYULh5B2WDeLmwaZPsNBDoe/RuGhrKr3yrIU\nD1G1New6QxlqzWCeTq5zjHlepnRGyRY3dzC1BWOU9PjqUh7Fmxt1dv+AnIif\nqZShoRmxJZzK7Evnva3LtP2FvmqUFRRvJalRoPNxtGRY5HNNeA4hcVUBQ4ho\neCd7A3r1Yz2I1vZa/ePtIj+TDlats7oJvasNIEgzmONFI1wsInv3NpMa3ryO\nnjDVqCpCAruVilsiV9JQIwUsMX9zcIAuIk7D3v4ZlReFx6oOc5BYkbrKREBI\nsyTuFewf+z5/s94AlgyEWorOMM+V1QrYu6V8323P5MSPGGUBH1ozZHRQd7cS\nuayuE3/6C70RqWx7+yV/c9fm3kE0w79s5gmJwHMBrP5HyRmXaN6fzaRmE9yN\nYaQuu7a1Q86myUJwJjHRVvxiKFJszC8aobh582qb9yBhhSQwzKirq52iKXPP\nawRZgxEMMyaHzNHcpQqTCOStsC8Uo1HqBTHpTwrnQoVebwPBYuLdB6xj6M5g\n73xAAksjh4HtioZ5nzpEj7Vm4Jdyal16N2tCYCW0Fq3LfY61FTi2XSIbmz2c\n0cbhJ8+MiWXvLuNxkxLs9m111O5UxPBgDDXUbP1m3KIyB1L6H2bEFm5owKD0\nPtgK\r\n=c48W\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"832820c97ccc398302d82b0ff7374dbefca27f99","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.15.1_1592392444629_0.0937212387454418","host":"s3://npm-registry-packages"}},"0.15.2-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.15.2-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.15.2-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"696b495b782149c1e07edddd3fd0f5d4c5b36dc9","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.15.2-canary.0.tgz","fileCount":27,"integrity":"sha512-09+R+Ix3u3/iHtd9fZgj4T76F2r8us0KWc1dS0LQhZMHzllAB9WQreeRPUvHVfJGnIoQhyrp5058xOog8WJbNA==","signatures":[{"sig":"MEQCIB34wSirHPTsIgEE3IkGNJDM4KQI9ddrB98Sn3P8hi4jAiB5PP+5yd5IpJt45SpKNnJp+9v+FopQwV3r0w5c2DtF7w==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe7aI/CRA9TVsSAnZWagAAclIP/jPpORpKgABM3y9e0+VV\nLe1wU5HUp33NzTdeYlgrYm11atq6YEa3uDaBqE24jChgqzRzkCLP/hJrg/FX\nElB3fz7gkOBAW3Ei4jEbteV5bjMCEFg3Ygfbp9XvrE2mkAnzMXsE9gxqSPZt\nRpzTn6hy3+H44LGLYppKvLMHJSNGZqaf6VN/MOeVtl3s0dgqDY1+eh1UIbi1\n95XLKa5Kz2S1z+EOISd+t6fCmqDSBtevI31qir9IsDXZa13oFkVyRBaMspPl\nniRT5YPsKMEdBjeNqSMbUzUEYFXBYoE9esK17nrB4f3TklmLOTJ21kjjEg5X\nR2J5FbmkUPMgBCoCCfJAWr5akMozI6xgEgQ0nCNq5rTXE9lal56gZCi/zj2y\nzVgl4ntHUeNWF9+mHi2S8Yodg2uhWHZGpPLe4JoLPHgngIPnPLah4FDUQyst\nRv5aDqJN1tr2XA7uDfI3lwoR0igp9PCHyPL3L3i8YSD36JKwyLKMlSl4vGjX\nEJoZTF8wxvDfDJDFqIXGNtF7HQVDHdN/124+Jcv8U7y5FK/kX2Uz/mibYVbu\nsyXzr3y1JygSWEubneFXtzgmyPD5hKLa5Pv1QwkiCeTpAYrih58XwZRnaV9S\nMF69AewzjDKGKa49FLRFCFXyqvigAjaZfAutSnpeKpovtYMGfPDbYIAiAgkN\nrV60\r\n=WtVG\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"18611963532dbd9149f2352404437832a3172f92","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.15.2-canary.0_1592631870922_0.7372713385504217","host":"s3://npm-registry-packages"}},"0.15.2-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.15.2-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.15.2-canary.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"5c57cf93a1281674a4e442ddafdcd5181d069be1","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.15.2-canary.1.tgz","fileCount":27,"integrity":"sha512-WUyfS5qrSLg5oNp/QF/E/Z6+WuW+K3+NGT6GtTxdJls4XYsnJbDECDkNQlYAy5PKOfla5+8ceOBt5LUFkCxQZw==","signatures":[{"sig":"MEQCIFRvhX0TJVkjjMqkOyO0HggNbRSccTpoxoyOR3aSb9UoAiBAKOEW+eAozkf0jnzv75MQ6DnOvFZmHFZnSOTVpJhoPw==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe7biECRA9TVsSAnZWagAAJeQQAIxf/bsAaiqHoCtasc5b\nUGX07PQty8bSbJ+30Wg4CkzDnurbsFGblpn+pcu7q67o707Hn7tFlYJTUg2x\netflNT9oAxtRpS7G3V/54Eygr4oOyDyln0uEGtOv5zHzmEZ8zyBwdYLZKRzA\nXq3lIqNUSvd8O7necfc4aibyrFxmxildXOGmcWWIbGBDTsplqTyX9eH5S555\nKkrGH0SuQmuEqDZ8PMDRy7jIGA0TeAyvojXL1g8EdSeEcTlKCQvLI1gEQ7Zd\nGOjmdS8DG3aDRU178xOW77+8M48Re+CedrTcQJ3itvzH0djhjQwRyoWrCIA9\nR7CA71CJJxe5FL6Zpq6J5U6aQFPv21j1dD6F0Oo8Q50ofHwBb+6x/LiHUeop\nV2rSYEtP372BCS9xUeBBfJuoz6Vd4QZGtjDw4IZztBlk7/lEtnZraQ6KSyUb\nQBpNtJ567zJN8E73tpMb8P1g/3w/xnhg851AGiTQgKVV+PGbaccj1A4DPz51\nmqzqjX7/Gg+au1LuLWGDzAB4UgYZGIWUaMNTwWXZxKI3zogsr+rVS/MCYGN2\nyFioEEg0jeMteKWAM9FvqLOousbZqmAvyxHjduCmXELkqBa+nJixy4V4faO+\ntx0+oFkVJDJSjXatAAlO2p/TiiEuSuSkftZZ87V0sVBFe/9i2rxTB3M9Ci3a\n8YFQ\r\n=e4kp\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"d26a287ab2a3809acfa67cf3e1e7b7944b0a3e2e","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.15.2-canary.1_1592637572350_0.9175617085616212","host":"s3://npm-registry-packages"}},"0.15.2-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.15.2-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.15.2-canary.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"735d0c55208efa61bf5f6b0a2bd418f9b13f43b3","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.15.2-canary.2.tgz","fileCount":27,"integrity":"sha512-FxDNeTuicf1dvllEsMvTV4I5QiahnQ29MgkMlxnH0IBIeqD45/cgwIrU39/1HjchE8dzY8zltY1heQrTrQunvA==","signatures":[{"sig":"MEUCIGp9CQin5a/54/QHuq1vkXlrkJxhXj0vaP+rBy8bmo5fAiEA7sXZeMAYPU6UR30GrZsD3eyPMqmMLx+dgQJyVetUZ+w=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe7cq/CRA9TVsSAnZWagAAlusQAJ4FE6PgJgkHi4IJp9Ly\nMMFc57RynIkUDWdOcjFKuEybu+RzYaM9zb2wdBjC+/g3TBTS/lagIK2MbqgA\nOp1pzUgGF4A3eFPNLoGZoYbTmeu0EW0rJEWJ9jc5s/VTp10YSqh+UpvSWUPv\nDopEX4FKRiVUZYed7+ieSLQwddai3c/MhGZzN0u8mMFU/k1G1m7/9hRThmD5\n97B2ScaMfkDw4VAGwIL6ybdKsaaHucvAerFONHELLy7AZq7iQXb33OtHkQqX\nAiUKUcF6SmI3NXF7KzijURs59PoLUS1KT/YA6tMt2fcUmCNYqeWpTsrzL9k6\ndX0HRpDWl7nvrqmjH7RRtCo65XLk7KxGLo7o1/1U+oNaO+dqX+5FmD7XL3LH\ndsdkTXQA9SeIXorhYnaG+Wn/ADqIgDBTCW1Ve07vzX1EBNVW2apc1M3G8pUu\nMwA4pAlQ590HRfmUC/sxRmm7596nVK5AxatHyeA4HOGkkCVQShIqA0Kf7v2O\ndtyolpRFOXTg21JP5ODrChkG/E9wFRa3Lt+ykdKsMpR0zGIQd5lwC9B5IhW5\nMiGhC8Cwo/z5zs0dI+YIgUsNGAqNR0OME40/aXBm13u3HJgHTbO0uRk1hgyh\nwn1YQB8eDy1V2HNlG9TGu0xhgs/K+JNLwi7/jI+IbiY5wVtCn71txcLuD23H\nVnwB\r\n=Yok8\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"3d85f365dbe5b0b624e232a2d1a4bc004e676a2d","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.15.2-canary.2_1592642239440_0.9452628938387131","host":"s3://npm-registry-packages"}},"0.15.2-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.15.2-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.15.2-canary.3","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"840309f9b63a0f8a13d26b89b7b7cda7b0ddc759","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.15.2-canary.3.tgz","fileCount":27,"integrity":"sha512-qLCxNXUifpy4X6hyEC8wbqDFBzgcyO57qOGEnmZxk2Of7NuId52ySKMfjQalQ6zUYM8jV6GjAsTD8FbYjr9W5Q==","signatures":[{"sig":"MEUCIQCynp/k9PU11iAq6Uc3DASzDJv6ndMpDG+oFvQGvnJ3eAIgKQSIC1gPeST/RxrGSmxltR97bkgIwQcEt3roQgILvYA=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe7c2ZCRA9TVsSAnZWagAA37IP/1amhcN+QHKTpn3LYTqo\n1X39PCX6QRE0MYBUnxvsmax2zg/aIdLSkA+gdJTxaZxdc7FN+PCYazuK7ak3\nAIV5kzq3rtbkbrVXLLWKMIAb5cNogd3xk9aCcnJEShKRY7X0PPgAlPL5DpBU\nrZ51hZ75mclA4P9k7Mt8Wpz3XkjAHiGNrTixKU7PK/1ikUgankT7xJeokU9Q\nV3UKMpM6gQHPyfwxqlNj1nLQ9jfIoUvNmihbcKBehXfzD/q/OAfobU9+TRHH\nxWCJahcY9ijYqpYxPFzkQ2ewvQQuvifb3xl7MGpmbhWigUapbb24NJDBTVp5\niv5JKaSYLL/ztufZUgM2lkirCL/3n3mRgqtp5lv0awSL0tE9MrrtErCcwxKi\nB0N3oMqvR72nb5J161jnei++1N+Y26ISzmCvTOUFKWA6QwXgdQJ4E++Q5uIr\nnSL1I9UO0GspKTY5rLZMtzhmPqIly860q5sOXpH9y3bKi8u3Y7aUpHeijxvh\nWQ3RGfwptEmS5TT05CoT9EPZ1bIov8MLTAJTieALK6tpcIyIf0tEVh455TPx\nEN9eX7xYHbs9XQgeusT/1ggTrh8l6WhcgGBRMr3tZwnpWqqUiQ+czxScY1Bz\nNnSu7A/O5j/CadlJB1v8Zci/9ibSuayj7ixX6ckt6QzKqEPc24JtvJ7ckcDv\n4vIO\r\n=jgHY\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"0f70020312d10f2f141fda41273109c0e6c56a9e","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.15.2-canary.3_1592642969032_0.24104083324110004","host":"s3://npm-registry-packages"}},"0.15.2":{"name":"@blitzjs/file-pipeline","version":"0.15.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.15.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"53aa635c8aa3214cb3ac13dbd888355343124620","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.15.2.tgz","fileCount":27,"integrity":"sha512-pP7m4+AdzwWisyhWnseutuNhKatc/DX0E85LY/b+VRKMKK/uWci5XkGsjwZ8pr0qKi46qKIo0+K4PtSXgJtaXQ==","signatures":[{"sig":"MEUCIDGBSKDQBwoUljJWagftTXmfnZ91cHnuMqXdyVy0SgwwAiEAz6Rjt+1rP2X6v3QJ47HQ5f6Kqg9sote0pNwm0MsE7Bo=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168845,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe7dAnCRA9TVsSAnZWagAAhpAP/0HCWFVwwEVzbk7vK4eI\nR1SETVJWGvZC+8V7AmpyQmIWR9N14Siw9nnqlIrB1YP6kasISUTv+Vj3oBZ/\nmuYGyDUH8WQ25jMiP/Kjxi9oaJWH9Ti/MxGGZ5/uGSFCNA7lypPmE+wy2j4I\nwUn9m2libBVYzT0XWGfWwpZ2N2pMiq17qs3sOr7VXtG37RZQYa/zEl3VNEsT\nmwv+7ZnWcVM5AXHaqPIAOoxoxCC9z3VDHm0B0cf8AwIZ+674PCaLrwzUij/n\n6U3uPs++Dq6A6aJg1UIZEBaYM57K8pi4OFyJSUx7TyoTlGQCadUmYPgeDUkt\nJ1FKbgc+5O0PnW67kg/Eugf4PqOSOhuTeZEA+fSDcJJms7A3rFmPmUYwMXA2\nDYbtLuKoezMqzpG2fCgMR/6OZTFl0Ie5b+BIlC9MaKRe7MQMM8EXt+O+TAw8\njAvLPC3COwfx4MCOduqY8mipwwZOwCQk2iZcTmeVyDetMn1P+9INWuLhzirs\nQnxt4COHAcM4cIsA3zGE78qjEvwI18xoY+74dnRw7xwFdJYFDdKpmrORgGcE\nyqCjchSgsuxcn6ywZAZlJWwS9XJ3OM1R3B821Aox/EB9jG9zynNKZE0UUFMo\n516U3av1tW6pzhvn3pUYWiVmNebykupf8uPkyYGboMsjzUgAN+n0oivi9MeQ\nHVv2\r\n=c5Cd\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"7a1b4b40d1ac6ce9d0736c8119d1af39f12f80e8","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.15.2_1592643623519_0.3802534132851514","host":"s3://npm-registry-packages"}},"0.15.3":{"name":"@blitzjs/file-pipeline","version":"0.15.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.15.3","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"87acc9e02c6e42a92073e10c4ca07d3ecf900dc7","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.15.3.tgz","fileCount":27,"integrity":"sha512-r1ZK2Qbkk2zxqX7HlPTpALG4mt47CYbqOs1s4oEOXGLkmRloM0fodVfiCVwVxX04gexzIVquGwzCxILOzLA93Q==","signatures":[{"sig":"MEUCIG576SmAkc2mUygNYCd8N8YJgrXc/Dx/wR6+lEFO7IFCAiEA6hkn1im6uL4HzwxBfld4j3oIwl5mFWSnTvwX2xy2aa8=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168845,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe8gakCRA9TVsSAnZWagAAfagQAIosce2Oa8wkyDj35Avo\nQc9F6BjidPnZWMbSy1GVS13XOTnZZq0ccSTrOiNIgaPFmOjDRWSa7O8IK5H2\nm+eA9PBBIJSyJDi5rei19NE46Gy4IKvOaDzGOAaue38oMTAdyOnVIIv3altG\nctAyiRYeU8371c/j6DirdR04IQSdSYslLamE+M3ZzeCgYa9R8DmUEmBLte3z\nngOnq5AaSbptaEUyz2GN3DqA/b07FBA4vwboTAZJ4CW3oEliYbt1n+8EjZR2\nvI8t12aCEOg04vzy0hIc2zsqyOqsyHrw4XgGjppcyCPaN3aDzyNn5EcmKtZy\nmcnkeQvllviRyZHrptIf8ZqYHqg7xEsnKSpkVFgVhpoQ71/jmEVQ4hsuDSkB\nn/L70Df3HZULGe97A6tGnnb0DVPw0n2vhl6WiKyKO0Tcplsp2MWDU5kvYdQj\nQbedaP5eDIFp0h+QhysrVYHV/tpZVXrjrSEFVh0QAv1u9Z6ykAGbLPzxkHkr\nn9u8dyyRNent2byrMD+bHz1qVy9eeYwb1wwViMFKsxExUnA3HQAkvHN9yCJv\naETUuoPLSvtTZAxiJzPvlF008pcS5rkxreUitNalSyaJL1Ig6D+9NYFOesVs\ngpyd9Hcz4yXeu1lVAUA+fyV7uaYM/F2SImRFWvzMS79tmHmeVqEyZp143cB6\nyEXH\r\n=z0Kw\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"9bb57307937a9bb93ecc761e2b79b5c1a927dc93","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.15.3_1592919715581_0.9820254545081111","host":"s3://npm-registry-packages"}},"0.16.0":{"name":"@blitzjs/file-pipeline","version":"0.16.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"ad34a2e62fd89477d4d297d3108d06371c3cee6d","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.0.tgz","fileCount":27,"integrity":"sha512-5yO81dTfIjrgBRid8qw2g5nKKu1ENDnTyyhnWT0fbTp/8lVQQe/L9ecqfdPExk5KKB0unIkXMSA+iWUMYe6EkQ==","signatures":[{"sig":"MEUCIB0Z/3h3UlXNzs/W/CNYG84683kCEbFsJkeKpjUBYWSNAiEA+Voj+O6y3bPsWO4h7nVw5lCQ3YOnIR1beWQiNfhWUwM=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168845,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe/FffCRA9TVsSAnZWagAAhHkQAJBedNpCf+nfcQzqLT6q\nMO6WZx/ICQ+lgZJIw5qo7r7KM91/1HLq6Vss1IDL5GkvZ7hTf8d2q/EbMldv\nVr1mK2zEwD25AGhHgHrIMP1b4f2dQqqii3VXQrToABG41dI0La0PJNeML7+p\nCv+OdeIdKAYxfNkxaD3v1UhmcsduPWO/IeRGF+G/miDNwqCJXgYhr0GaVaPv\n3kV6W1x+AFf2qvgt5cmQcnOrEmRwwt2+pPHZJ0GN0PUdFNiuRwYyK7GDM+aJ\nlGruy3V4ta/62cyZqZI8XGkDT7FjHkTE7RJQPcoU9/uKNBEEhoVI+YjwxdMg\n8Ay2mMjvhF5UdSpy0GE0a5h68ODiMbQlJqHtMWuqncPLZVBnMvWWrWkQ2YDR\nyO88P5fWf6mwYW2eekJy/t5xNABPRJW44jdMdqDhQYcD5VVt5DOGiCnCn4o8\nSFLsrHgNsIgBmKar2JR+/AC9cjgNg4A/HwwecclY7GSKdms+3DA3E513Itxs\ngxHb45zAZEFcnhHNbJlJqZiyeUcKQQVq2tsYS6lPVnG+OzMbMS0Rs0BuUbCc\nbcapqAeJrFHYvmTFCDX7dGgIS+EvEJe1ItPuz178qLiwMa5Qmo23Q2R3MVJ4\nLp3w7kjSYfeE7fH+ob20wFuItvLKwVl2dRuwsmyxkzRXx9VCmDJ6rjojIW6H\npbq+\r\n=lBcI\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"843dbc2c66bd9cc61e883ef7ca874a0ca370679a","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.0_1593595871278_0.19405145794121803","host":"s3://npm-registry-packages"}},"0.16.1":{"name":"@blitzjs/file-pipeline","version":"0.16.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6a00f32d070c5c931cb36cc3faa9d99c5f3c9016","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.1.tgz","fileCount":27,"integrity":"sha512-+hRsJHzoVMwLWFsA5lFoDLVdtCHgYkYzdRHJREtU+VW7z8wyCfbtriXMD6rHsN18dnBDjbqVocgiMc8XSPAgwQ==","signatures":[{"sig":"MEUCIAJWTx5q5gp6hy3VobWfJ8gL5BnpfBIXj9iGK4eteDemAiEAnYaYuSSLiTxUwqLoTkazB4zJQyvb8oFSf9SMJfniSEc=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168845,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe/hOkCRA9TVsSAnZWagAAx1cQAIKDVI/ZBaU8H4T+OVIh\neoU8PwksGP5bBQRXok8OxaiPndSDfsSFqG9qchKbQ6dBapYtIbyR8cwISt8k\nuLtcvMriZe48NOb+QSIgEx9t+okW+Px2Xp8WNmMG37wfjrr3GnsHSemNFkRi\n+vrzVHNxwJ+wC+/hBkZT+Vqes5dsLvEkeBxZac0ufAaTWMuLD9z3edN+D0CT\noxcjQ4FexoCZOwQwgtxpWOS2zMJ4bvau04VCZwTLx4+CSy+EwuGA/NINB1FO\nGQHfWmwc233iNYUOzglRL+CXHgacUgbyoK14MFsZ6U0nB3Mr0NhfUVNrTkpF\nNfDwVvaLE2x0tLGOmzSLxAqknhBORj+ZIer8sxNt1eKDWeuvB4n0gdXbHO67\nJVrQtzyEE+73A2h3VkEV55ncJIDb4Y6kACxFulrJJll0kkeDAD9rdu22ggum\n4lO5Ji2oEDHF0uTN/EIX6K4RcgZOQmR4gk7FtD4hTfNRAF/4DGeVbx4o5bBN\n4gyXczu6cbatMrtJj5kKjlama7ukvwKgEjDRlgpBT3uJFp8mefOB2fuN0tjw\neYo9UF2WTWflJMP1U1j5gHk3RPmxsN2klt0A0gbivq/giUOj9CAg1n3o4NRE\npfbMVYopbPVQeRJVzVYVy5a7Jgw2YVMqcw/2XVu6YoyK8lJiBLszxRs0dn4p\niqTN\r\n=h6vz\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"93f476853551899d9b151d837b03578151aa99a2","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.1_1593709468911_0.7230090436721521","host":"s3://npm-registry-packages"}},"0.16.2":{"name":"@blitzjs/file-pipeline","version":"0.16.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c3c11e107163b6f86fae307b4f89f82cb9a23137","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.2.tgz","fileCount":27,"integrity":"sha512-wClZqT8wQxT2seXIN0SnpviV/gCjQfObNLjnXcXb/fapTI9ZZ6r9xhEcFwhDRYfFzom9XGbtgqLeeBvPt5J97g==","signatures":[{"sig":"MEQCIGu7DnTH01wr85lxlCXGE1WdpwpNOevpSxDWujtDCN5SAiA6dTph3wJR9EkOjmTalQdHTGvKKNoU3uqZZ+gdc7iFmA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168845,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfAzhDCRA9TVsSAnZWagAAJ0QQAJK/4rK/TShMURj1CC4+\nnnArLPRuuZUdhYDp/OLuwhiOFn8fDogC06yfmYTQ9Dl6nPB4X4flRaZXwIwc\nToTxLVJxB9z5eP2vmB6aEB9S/2x9OqIM4kCcLexMMR+vTPEpb5RBYxk1hOb/\nEdV+epWHTQFEoRPYKfCxXrT3UWQJD+t8TWr5WzGT0epvRA7rO2s7njNcc9ny\nBdodHUfuNjdyCAfXiisF2PrnmlVyVXxfhXF1kJAPaDABIlWFjyQFLi/5wV+I\nJ+r/eS+VvSQ8c3f4sps90g+gjt9cNvNYT1dX/juPar2NbX9OoKGCxS5PhY0B\nTJ9saThQdfQMW4oKnkF2+FxoZpPllyJGxrtpAgjqbe+y5NcYlgdlNCue+cbY\nnPnfDoLivgjG89wTZUQF/vzUeto3JE/4Hwj8AcXs45EOx1f/llslXMN+ccdd\nup+rrli8MDof9+uft1oww7KYPMByj/O3upSOaYXSV5szCiqomz8YrnWj9uuN\n70fGXbCCXHYfzHUx935MUsJ3yoLNbMHslT8bCpoGmptug/eldo3o8ygttFfA\n2HehDEPrQWt7Zo/BfPJAHslJ1GeZZOXYM+NvlY6ZZDOFHO29Ofybh9I+k1F/\nRofEbG8sB+sQFHBuIaJ2b2kZblITMq6LPKPguGa7vQiNPd8HcHNIChjNvTX4\nMXAv\r\n=CNtu\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"440ed3d81a2e80ffa84ba720687e538a9977fc3e","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.2_1594046531092_0.9146920830387613","host":"s3://npm-registry-packages"}},"0.16.3":{"name":"@blitzjs/file-pipeline","version":"0.16.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.3","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"b4d5f923c7a4a2e1b4bc1afebaa392bf022daa02","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.3.tgz","fileCount":27,"integrity":"sha512-MwWi6rnHO7INGHdTrDY2GODdJvjrhc9kwhODtKQE4QUMeXrFsHf1ciWm5GQb+1qFipIIqS1FlpH0znOw8nwWqA==","signatures":[{"sig":"MEYCIQCP8+6PNpazFZc+Qw8z7o+8tzau4JfkykG6uFOwNsh0aAIhANJ8v1k6tBsiL+eXZZt17ycSP9I5MoH7EE/4VNGtqWIU","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168845,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfBZ1bCRA9TVsSAnZWagAAxqoP/1GKxB8YyhqwTzx4dTw8\nNSPa1QKFUGiLNqxEzP+ryKe6n/SmoSBYK1YP4yJiHCUYIDGiathAcajrqxEa\njN57ON20/A+8J4ZO8YybDz90YnTBIUFIfS1thjZov/3ww2onqILXXwYgmNVY\nQnn7XOSH7f9Hv8MUVWPNHAfJWjsBauIClvpip1weSb88R1OGqq93Dsp8reG6\n8gtpU+srxwiDTIGtCHR/QUwe5sYDYc4XMvzCCgC69P5/6s0AT+gI2Q1xgTW2\nZK/gEVgMqhg5/BzPaySPUWRcRprqfjpZR/su+drQjGp7EV73jt1VvPEeSMYM\nRYEqNOZ02hDUdejB8Jfe5ZncMQn9cg1cZiHGGhBiVpDOMIyU7U+8/eyWgqYi\nBrj9bWOpGJKON3pXeqoI0xc6w8eamrUNvu0OrbaV+b5bc6NyIJgYj4fjehP/\nkYisX1ttWf8FtM2zv+FEaHKQHZ4r7+q7QOBIkpmUIIWtIYW7Rtsh3+dcp6ia\nGF4a2hyojezf2GSHmBh5eHQRBGYXayj0TMGb902RVrQkQmBdyjYO/E9deDul\nHU3P72//VWf7C3/pva1hQG1U2eqJiCRRlXWyKaQUiu0vZ+XQPu6qzRh6MjLk\nbHuKFqRlZXvOOQqWs9uImSWA3+588TSBOaI2r4cilQstPo3VMBoi07kQkKpw\nhfVU\r\n=TaH6\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"82fd604be4c4882a6263afe64e44e67e270e292d","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.0","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.3_1594203482783_0.9691513034105228","host":"s3://npm-registry-packages"}},"0.16.4":{"name":"@blitzjs/file-pipeline","version":"0.16.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.4","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"e1631e21474c9ff404b11a39d73e80f753cdf5bb","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.4.tgz","fileCount":27,"integrity":"sha512-M36ocFyLJjsdT7PsFzQWcfvKBIjnPWzw2Yj5db1PCSMjbOAxLM3tB22WRzfu3gkA7sQzPF2Z0OaxgOvRwCQWJQ==","signatures":[{"sig":"MEQCIHMspZuGGDBkbvN/4njKiurw6VtxWO5zzEruv2RZpS5jAiBCTTzDsQzslzhHpqAPCJBEAbVIRxCcQ+QDGR32k9lVQw==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168845,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfDFTACRA9TVsSAnZWagAAtmIP/iZzBOBfMiWXUyvDidvp\nsWzYQGSgBEdKjWUJ60HWjZF+ZKPWQHmV3ieDiZknsTgmk3rOLrOM1oHzg8mR\nSYjpd+dIh4GP6p1Aw2oxx/SGlzd8xPzwDir0XPX0DnCLwZ7bDHGI9K9XN3GR\nMnP8dADPSaT/ccb+qSZyAEAseNweI4mPM5GWiNE6rYbdZNmOWxdrTGZJkUmF\nCRPgNWYknYlTASUopVAsaaKoLtms+co1VBTnc9bPhu25Y4buhui7G22AcMRf\nr4LGlgWwtLpgc/4QA0QVYS5FkA82ganO+P742pMjegTCUckaprZJP+/P0MEa\nVrblTAdso5/Np4gitCpOtTh9UTdYq08/K5TvNT4TYeEf1bsoyogkr40ioCfv\ngpex/rFurEgtXm1iikVZe2fxC1kcEEfxsDiaaYfwdxUJy4BkjQLuL7kDbb8u\nSaGGeX6+Ev3NXSZ38Y4prqQ3YG4QA0GvQVnvwtSQmSGUIQZq4bJkNNox7w4h\nkXGSjBsWaj6Kji7GwvQ2YPaLzQSLTc1si5ymp6Sg3iEbjo2c64G8KYDyUfSx\n4VwqMqy2p0y+PH8F/lyvOqg3Q1JRZZZf/GB/natURb17F2c7Z5GrEb6bhwNL\nbc+srw+Rrc7EHpkuJCsinb8/7RCfCwFop4agiHPdwlelsIV3dFJV4L6MUlSi\npRcL\r\n=uKY9\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"02d0d06b8413c9f77180c9a63a873c900510c40a","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.4_1594643647903_0.23078787407476264","host":"s3://npm-registry-packages"}},"0.16.5-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.16.5-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"f7354471d261fa508c3cb8cf38569fd852616442","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-canary.0.tgz","fileCount":27,"integrity":"sha512-bpMjUX0XYkpzk0N+gmFxV/c+o12V1gZK3hW6Cy2odAVdXaTJ+u9d9Emk2FRb1P2s5LbHD9V4/US8A04T07Da+A==","signatures":[{"sig":"MEYCIQC8kDZIUenoQ0ovATXAYOENDNgW0TdFDwC+L1MgAM0wLQIhAINCtbUI8isrxqKNRqPeLNyOK+R+FH5oDNf5nBn53ydO","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfEF3sCRA9TVsSAnZWagAARlwQAJkS050w3r1XtG3sQzqo\nwEGN6D6nki2O2QliFKMmQjbjzc+NZP3wFuT+e6RHKEamD63GWxwUOTfZYcFN\n3QbnX5+d+eS3NURLBqebFllDcX2zYjPbw8LXiGtn0Q1JZzDoHBOlVYnvCdW3\nS/LkG7gJbqi0hr4WYEHeNlP0kMUDkONQjHwPtvmG2f/aD/fZvTnAsQ6yXtCB\naooci3JD16EO9STEx0YIppklYFodN0947vT+QZ79KYXpFb3ECMU2eq25uaCX\no0hklBWnnECt6oMroJCvbE3xn7xwl3ZaE/ry7Xrl+ABJJ2ufbnsV6zYe9hZy\naRVtRj3H5u6eKihzq6l8ESyKYaapec+LGqjAAbukensK2lLR6aOIIeiiB0YK\nKX7adSCVovP4DVBi46LTyvQTQrHKEJZCFDmjapFC9ODxddQ8w+1E8GGuCaAK\nGKu8TOgCuq2zRD1lQ8WQmj49I1Jjr+0K9dGk+Lq2JnpRQbMhAXZVZkqljq7U\n3OOjDvFL1+vK7PpnLQ+PyzQq7mUNS0nS4PanzFJyLfjKFRdEuaTJLpxwiPx1\n6k09QQ11cla2txEFPtS6MEP2YIjknVSzFBZ3BEtPaLvrHZiDGb/+FPDDwLZ6\n77Ck60uikvZzvW3kNPuRoJN5MFVp0+dT0IHK0/Oovio7iDfUVu6V/lOpia9u\n6gfW\r\n=FLqV\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"2e7f191a316cad4d83dfb43c452271041ba26759","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.2+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.2","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-canary.0_1594908140434_0.8075313219191083","host":"s3://npm-registry-packages"}},"0.16.5-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.16.5-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-canary.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"4d8f14749ed5d7e6bf1e61fb4c11c6fd60bbd4ab","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-canary.1.tgz","fileCount":27,"integrity":"sha512-pKml0+pPysvvW042aAtnJpfvyOa40EPklq7KB5H8PquD+A85FFfJP+iT/BbdFVHFV+aHDK64BUEC6wtZEbVdBw==","signatures":[{"sig":"MEQCIHfg0wVTEU5qcDg9NDIeiDGpqa1GOfOErY8S4JlNVzINAiAa+22IR8s1HrNFfZSdf8IuDn01iojaSKKwNLBtS338Kg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfEsEoCRA9TVsSAnZWagAA9cwP/iFuaVYaxsYYl9VVcjq6\ngnKFR3xQ6l3Xv8DTiojFhuc5FG0fcwzZyeF/GdH9dwrn2MwFJJCUdLJ/GZx3\nUMYUijGjgCtZBV9ObOq2KIzPPWJKCPTLlRXupLV5FAeOBc0GzR2UtHxZ7Z+d\ngmXPnYqBKqYpDSxzNNI7Z5CBDItpnVw4vvEklHToszxcIFAkj7TqUj3q8HLB\nc5yRwod+jEbA+Lh27hKQ21kW853qRDSqiw9k2zfCLGA2MRW9NdiMzOyxVOB7\nDkFcvZRg0j7frJHpekMZgeoT43nLHo73J2EahRiGBp50pTO8lFwvciYatDyg\n689KNP+7N+rMS2o2IEdqUX8kDGYjSoRv2iioYTIKfYnQK58/DvF+FtuX9gtp\n4Z6yijtg0OoPoZKvSIccUQdcZkUF4nJ4bNbxjCW6fStal8apcw4inqs1Wh0k\n/Q1jp4E3K5n57VZeEb0vSOWeUGE4xh70sMV90LMbQwPV4f9le7u+BzLHn6G+\nKEpLfW0Frko2aKRSUrI/YTUhWoczMtxKIhgAu1VGN159s/bKWV3TJ+JdIZPB\nys8Ii8FH4wNFd/79broGTkFWXMhzCqh0iuU3U63VWAO41k0VG3kCxQEyA0fg\nigKfIXhbNgqyq3j28Q2Ct0oB+pdxak0xw4w42oJB13o6XyWkznXqRGwy6b7/\nuUOm\r\n=yiDq\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"7f232be90f79d616016b11ef55d645f245b29fd4","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.2+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.2","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-canary.1_1595064616279_0.20112586244701913","host":"s3://npm-registry-packages"}},"0.16.5-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.16.5-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-canary.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"44f497abde269d41b4767300245de68dd4cc1e55","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-canary.2.tgz","fileCount":27,"integrity":"sha512-hLBlKa7a0/qmMCzcMZDLjSIZn5KWjh1GCqQkbCYS2J1z7xC5iOsyRjQxmEkJuziXRgXoebLpnS2Rgr6YFs4Oow==","signatures":[{"sig":"MEYCIQD4rqrKxt6PV47GR+I1/Kq8x240NWzl35isxPJ2BwNeUwIhAMU7y5Q2gVGCeYv3PuNFH3+J8LPFoUzvSOPvRd6i6I0c","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfFaCKCRA9TVsSAnZWagAAvw0QAJtOxezvfUDbn/HKzwN1\n/ctRkrtTZo5AwgaGFyQCxXPin9jZAZwmha6JsLQ+D+FkPSMvuZ9MMNl8aet9\nf2XlN1cpeEC95ySxB2HbY3Qu6uwztZB6YKIgk0UCsqBcJz82Goqk9DFn3WrD\n8n7Eo2tkRKFH8BpQO3LDgvMRh3/As6cMu9r8DgDp9yzKh3y92DsikWdNCivj\nLWFSRN2oHdmRm7XpzBsX92yuNZLZg8q7FEiJbixS22K5MSI9Z/oTLG0p+5qD\nP6pagjhHRc95lsdLuUktLBADo3tkV5YpLjkjFp1snznwguHNrkH3jVWq+IZ6\np3eQ+FR8Fbe7VNGjV5MnJE2Ls2ajdidu/lzDZlizTgWRCONYImQsLdRYLsus\n+HQiZ4U65FcZ8Y1POZj01GfXogE8FdtU4K/rC9GfzWgJlA3XTPwR7l0574HR\njKuIu/yv+DYEljl/EBwYAMpgNYlm/ei0Gx7aMrlqWArqsoTcY0S+4awLyFd+\nFnumCasQ5MPGQiYu239uHvYV2oESdiSyNBQP0JHtz00dGVaJnHNsNiHKMBIC\ngep/Kc0h9i/UJNZaQsvPo1qLWTKf/pCQ+CTEZ32t1eSMobCshHOGTU/6rvsR\n0s5mzxywz0oV9if9KDDnH6C9qVbTmPxdPyjiCOqWWIFnscpaoUTGlBegBLkG\ny8r8\r\n=CYk9\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"2d9338a61355866b1042cb6f6f07087024a1b6c2","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.2+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.2","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-canary.2_1595252874620_0.7510878249814215","host":"s3://npm-registry-packages"}},"0.16.5-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.16.5-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-canary.3","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"d7f4646439548c58da2c01b94c1a2bc48f51abec","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-canary.3.tgz","fileCount":27,"integrity":"sha512-/xv/xpJk1LHN/hGvdnCEOBSk9Pi/xvH+/Eb/PjH0osTvTznhZIBccKKjklBE98z9tKwI42teg63o6LWqGQbLGg==","signatures":[{"sig":"MEUCIQCsDfHi996tvMB7Mwz9J29+he45RwMlS3lwckaXh5gNdwIgZLfZxkTFUE/E0rDlDVagAIDXpD4rPe/OcgRRJabG4TM=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfFl6oCRA9TVsSAnZWagAA4PgP/2qBblvOnyAeGyH/weie\nak6doCEkFpEvJ+bnGmia/JuTJ/13B591zUYtzByjb+HiiUrR0Kwjaeyum/KJ\nofz9O3VwMIGzitbZ7YgaLUwVahDEJH0+Y9ZlPo1PNHLSnGQxmzeaBhHZq+L7\nDvgfIOo3c1BQipSbStDDTPqKtzFXfxpHHeMbXkGSvLmRMygzv6nGQxcqy8kL\na/Lc+cKsSd1VgyhXS97hQ61LyA4drttYm8kEXmo8qPEozff6q6SH4elMN6Yb\nlDUh5ce30sHOB2NTJEe9g7fIJN6w4Bxfzm+qiDS/YucPN8zGJwfU2RmbEBPG\nxBkec34504T/CWqTg0/maQ+lxJBloHUS0d5RQFTkMuDBnjf4ENsCO8XR0mvO\nyyzkNRmRcY4uIUExi2DiPT1JSUEk3WbYif+Vvj5MmftpW8RZzDJitramDVO+\nTMlOIE1t59+O6M/KnUYzk+udpacmfdWPMNIzEJpfF1WPFLP4VMB+K39x4WFU\nw/J1oiJ/BaPaare1k7IiVavNKaIiQ9c6gu1CorKSgYZb3+mS2Xm74xvGylw3\n4nt1fkVaTjjC7kq9kxhDBbo8stjjKYCdWae8fLXvfr1weNWG1PJmqReTkEF5\nLcu7jDYfpauYI0TBQgHyojausBM1EC2hW6QZ6qrdazvcTTK/S82jvcRTgFUn\nDm9v\r\n=OPAq\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"af68b9aa0a176dd2d98a1348017babf34333dd13","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.2+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.2","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-canary.3_1595301544263_0.08793488232017577","host":"s3://npm-registry-packages"}},"0.16.5-canary.4":{"name":"@blitzjs/file-pipeline","version":"0.16.5-canary.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-canary.4","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"45db0b18eb4c84514f5e38354c86653b3471148b","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-canary.4.tgz","fileCount":27,"integrity":"sha512-p4rDOfelkNzX/ccsFGdzXip1MEbwSGKbs7cCwhMPwoj45uBEFiv4nv7l99XgGmCVUmabBEKqh671XOc8nXw9fQ==","signatures":[{"sig":"MEYCIQCbk1+ftZYCyKIqoHJnXAqjwvtHZNZknBXUgK9oEZ7xMgIhANy5jQpi2QHDwR2r5SRC/khUpTE6GWOhdnjtE3+cyGD6","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfFrHlCRA9TVsSAnZWagAAGxYQAIownmSjUdGJQ9jVOuCj\nDA2nLGwuG6NfDEhhOjJ6tqEK/Y7tDUsmgF3ZMbEymRtjo9pWc13iAgjQSEey\nku7Cdxv8LpOZ9V/pL1pwUupjtkqxw8JacFFm+/ABVyksn+17SxytUY9T8JAE\nh+3XrKCqJHFvM+JMFrino1H7ERbalggMDE3dApXlD7+dS6/O3zjBKOdp7bX8\niCUuscWKo3jFxd9FDK4zZkflcAYlcL+SClhK8dXEIOIED7JJ2aaWKyUOXbPV\nDa52nXoLnb+kPJ83zprpBJLrmoZsfEHMiicrmpqfWSROO9TwYgtLCxJ6veDs\nnkOxrHAs30QndAQkgxH0Xv9PnPYanFdKK56z5owkHoTHnWes54wN1BNfin41\n85DbQLy3g2zOtqwS3JgOUxV1BuCbxQMyj010M5zOudcoQ2b50nzLoSUl/HE3\nEFpOSr4H+ObUbHBQb62OADakyJ3O1XeiZTy24fjNDt9gles6kByITbt97HfO\notDQpLmmNbRdeOjVL2A0IW3Ah8WqcVIfB+9EggO30VpekAfT+WZdw7zGWyfi\nvWYRK6WoxVZtiHYP2wdhbYPyDtZi/wprHJ5HWSMfLS8q3fZl5i18tQ+4Iynv\nEWXfbjDE+BoTS15LoNH1bHwizxY6or5UOx8gsLlMQXmUYT2O6/0lhjka5fug\n3Rpc\r\n=Ehaq\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"820c6785334d63fe9d757c513669eef07b7e860a","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-canary.4_1595322852622_0.0368988837527624","host":"s3://npm-registry-packages"}},"0.16.5-danger.4":{"name":"@blitzjs/file-pipeline","version":"0.16.5-danger.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-danger.4","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"41f1ef1d08573622653ba984ff426ea07ac6e0aa","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-danger.4.tgz","fileCount":27,"integrity":"sha512-dKI6cZTtxZo2xlzpcR9Hm5G0T0TuZOboUR8v4D0xfqAYrX+Osg8rFNOWETMT/rqRk6kb2UUAGXYAnKKcMx1GQg==","signatures":[{"sig":"MEYCIQCBzARiWWWiUPYxI0enac8yAkSG9E7tZdaNMZRNPdtVWgIhAOXtcwIlcsNVCIu/7tTqSGJ2P5HXtF1JCcCxlcfSNOhw","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168862,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfGB/fCRA9TVsSAnZWagAAzYIP/17EMoMwo4LgPj+TISzB\nZhyl1AOOGSba7yhf3vd0GTWZnGbwXGwH9N6pjHSZB/ZEANLG6x2LEIrK9xD5\niz/skanl22wXP6NvWORtHxwGyW/sfRSQYW10XhWkGk0gGxFp6CKI7BXU7mJt\n4agbxgMkCIurAFxRkBjPP+8C2ffmy7Re39TLYYzK7ah2plerrcmxBIMI//MY\npAYkgdMe7JRPab8mhSuod/uegO4zUrr9urQzKb+4mV1ITIQ7CfHvPCExU5Vf\nuUpTUKZxsFV/WBBB6JeDRjLkA6PdgJAHkXHQwF3pnJeKZkqD3D+3mj6azlZV\nxDepEVMmrAI3bMD0cir7RZIePrTqs2m+MD8bkX9lSYI+vQs6wZWIh9C/a7qi\nWHRuVZ2uK0R/qq01ECa+vBCQ1fxq89LC0wK/ZXT3Y61uODAN37IFMEOMToTY\nJ9RFzmLM6qvxy7Z6Z2D+4Re0w0v7Dm5mWDo2O3mxMkcVXq6mVWcTXutJcf0v\nBHZS9jM1nSsjwV22iFzJ/4VwPCg4IZ4L4kESaDyyQZRP3LG18dusmh24EPSz\npr1hFGu9uaQiUcFh0oAZsj0va2LruNQe3Yxe9Js6+5+rYJqEcyfIaXbzJIyD\nv2B0RLCZtPqlSjEbPbeZlRZTqk6xX5z/bqmhtfofFJjSvR0fnJy6dnLH8jki\nloHK\r\n=PQJW\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"2f53795f9e70ab37dbcae8c4360988fc368f9f10","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-danger.4_1595416542593_0.5006782100629614","host":"s3://npm-registry-packages"}},"0.16.5-canary.5":{"name":"@blitzjs/file-pipeline","version":"0.16.5-canary.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-canary.5","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"aba8d54c534f04fc473a9b5a6adc4c4753bad9bd","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-canary.5.tgz","fileCount":27,"integrity":"sha512-wEGpUMz/6f+/IgkaiYM95DpfHsOvdF0Sv71tDEvttvLbPXjtHLoPhgX7R85u+ArZOp5C2A0GABqpPN5lfqj93A==","signatures":[{"sig":"MEQCIHQ3pFyNiEJFI9yil2v1ul5xyUUr5r7SbpCRTHKOtrLnAiBM/sOH9aYTevXsoRHryRg7kPruXlsngptR+A4fGNUCCA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfGFnkCRA9TVsSAnZWagAA+sMP/2VsqwmG6esVu3p6SdcL\nzKDXLaqmO9yekphE+TLNFzwQvz3l0N3aHuBRM0Gqcq4+2O7YYRLJ2ixtLZHd\ngcPQ2G2WtCjvztfD/SglTl6+1icVmbM7wQ9zOkCyI/16ht+hzFlZ1Zs1RN5G\nG6vtb99ecSYUFLBjR+OKbFEedh6k4FXNcsSRYamYNiriFpyU8c3kgtZfW6+V\njUNxpV8Kh1HUSUSCTbWgs5L90lsicGuI4Q/GeA53MgASD7zpzQGHRMI8d4hs\nKSf8gIbeNhZe6t4QpWTBwiiaxiQXOyt3c+WwamsJhm8NyVLX3GwT77sdv/7a\nLyUQ6DWptMY5bRKhrixRwt3fGREkYNov+lpKlA3O9SKBlbBOsbvON0B4kAhu\nNe4X8n5F8iXnS8X1vexYhbarcuasCj3oHPg65bap08s6v7Z59EjFVstCGeLp\nkLeOr76cMTMVI9kS2lPNFJW5SZA3/KB65ciLFM5MoF/OJpHGOjJ+Z6nv8Iq+\nUzrIwhMkMgw768uzNKvD9bhc8YCdonNyzXsv2AZoin0MCRZq3KQcY87JeMfY\nlB8cPUgXPS0C9POpzNfowUX0IUKavxefLtCBx+oht+YzLPhFspOTM4D0Gwe0\nvQZUvffaseGWAiadVQoVvJ0LQujuSRK03BfXVq/h1BPo4WONDj+/hpbFb7Bq\n+bXV\r\n=cJt7\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"d8449d44588fb548284c9ce30b73484157f1d486","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-canary.5_1595431396473_0.4750255774280363","host":"s3://npm-registry-packages"}},"0.16.5-danger.3":{"name":"@blitzjs/file-pipeline","version":"0.16.5-danger.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-danger.3","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"902348e3d8523aca0ac07a98a9591738f55b4e7a","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-danger.3.tgz","fileCount":27,"integrity":"sha512-zt2s/SsP1drnn8LUlECEBasq9Y/TSrfdXGIxZbzuivydjGHbpAyb9RUEBemRIZNwhrkKIafkX9sGZQfhEYroIA==","signatures":[{"sig":"MEUCIQCmSP+RkRxNfTQM3TkHCNgTeSqi5HTl19EUHY6dDur7wQIgS2iT3DH5YYRNp9jfWW9u/6IxO645KEuEQLXBxnqDagk=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168862,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfGaJmCRA9TVsSAnZWagAAdGYP/214KS01Pnn2VhSUSVGy\n+zbMQyIWgSTrvV8uhuVb8k+qUOppGZ+N+yrgqgHKLstc2Z6+Ow2la+wDUqNX\nJBrkjVIbj2CyjnXu5kREo4Zcs1EqpGcRh6KKCxAERjPVtl9aZIZT8MHk3amg\nxxjqjmzdxOC3aUaIvqSlw2LK2UPL0a8K6XBKiOTHRkCbCT+PeFBBiZqynV2Y\n0SHLkay60gbrk/ylOMe8wU7PYmh4QjCKsQMJH6Lij/gBqBNWETzFQZ6iU0vI\nBD8ISRuRy01UI+mFaC1NLzbDBq74fSwEdcO+eF8nJOPvJniUQ9CkAYq9SpJK\ny5gf+FnJw6wNvcw/W1ETreKMUF3FaLvCHR5pYxqKj3kIYu9yCDEbLVxbHOjO\nmBOSerLnJVeD+cFAnpxduze9BQ6iUr577tN7ALtIjTxas9o4uPI7+tynxYMW\nDYfBNaNInZswUXZNNud8MwuZCUv1LzeXcpoW4MS4zCB3OcskB/8wcZtBnSsV\nvMjFMq5kXsjR9Bh3Rw4JkHsahyKzdsBDTlSb+NxAmYBA8zAYnzpK5UwjmuiN\nnhbBXvWBYmAr2jfXiKlnA6t2aaT3oifxAGDD2LJe/7ORDt89RPVyBcxCVo17\n6FcxApwBVa8Wwe3sToFUgDt0AtaSX9tL5Ijz0AbuhbABHUIDmbcf62mKciU+\naWBh\r\n=B5wY\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"9df93c6fc405138df17c855989936af6ab997781","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-danger.3_1595515494350_0.2638200833079849","host":"s3://npm-registry-packages"}},"0.16.5-danger.e654019.4":{"name":"@blitzjs/file-pipeline","version":"0.16.5-danger.e654019.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-danger.e654019.4","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"aed6b8eefe5a48f2743c585b73ffe8051890577f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-danger.e654019.4.tgz","fileCount":27,"integrity":"sha512-+eTcxS9Qq1tsK51vQ/xfrFZSfpIGIPqr062btTnxB7Srs5wWK5T60DTLyQa24e4ISAB5+aVOclEWucMMncppAQ==","signatures":[{"sig":"MEUCIQCdlOZYZXlOFys/A16BMRn3MAxTCqAF5V1cEp7PHrtU8gIgVFqDmkijM91LQIRAkHMes68vyJScE7nTWN0FPL1H0w0=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168870,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfGalACRA9TVsSAnZWagAAWrsP/1LupdgVwLdnHXk88Heu\nItvIaQpLDKwhr14iYgeTa6YgyVZwokS5d9Bt7GIdIrjgeCzY84Egmtw8lH3l\nSCHjfcyPsBP28zR5NfqzNYJb43RXu8HdDYjkm9uelBCAf06elE9AP4tAgT5i\np2xAj9tTcEqn6AAhBsItRLT+Xibmy1w+vdMYZDrb3VHufkDv6J+KnT9p0fJD\nSt4ccTRPJua8IQtTc+Z0tfKqUNyOif+adzrCdEy/bgBoQEkJvhG81DO0pIw5\nrSS2ZtAtYfrhzPJJA03C/lqdXI+wL3CQ6r6KJjod77cZ9rvIpb7ihgAfEjOY\n9EZEJ843O6xBOMhm8MMUXhEidzsFn3q+CHopAd6CeHQpvJZlrGteNQRHwS2Y\nDIjqvsado3aOtwYp4kEc9tLbsEkNQz2PBWye2X6hMSEJQMklCn8LXu+hhrGi\nvij98Xs23CMOazVNWqGEOueiW6kc7OmPWlpeFIuLYEdXjuVD/yAPlxWmWAHa\nUgL8lfMbWyg36wsvPT4Nek3pfrUGRcO+4pS9m/aZ9jFYAx9vN1mxzdZax0AI\npXLTEv7DrTm8nVBM2yEX19tCjDFihn3BBY1/EdVIljIg8CijP822YQYPYZ/H\nRz20uppT7vP9VrTM6Sb13NbyrPktW9OtNjQhgcr+LTRU+xw+dCknrRFV+Yw6\nyl/X\r\n=IG0f\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"e65401904919d8d2a9825b344b64cb8584292110","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-danger.e654019.4_1595517247595_0.8015069570165458","host":"s3://npm-registry-packages"}},"0.16.5-danger.027430a.5":{"name":"@blitzjs/file-pipeline","version":"0.16.5-danger.027430a.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-danger.027430a.5","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"598486b0cf848a2171516b1fab7f8d244a12ed44","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-danger.027430a.5.tgz","fileCount":27,"integrity":"sha512-6sqpn/3tUSDN/YW+DmpyliMyySS8g5B2AVjtZg03DDhPhoMker5qMyMzqatxGf93F15ijqSNo8KmVnF4xvXKYg==","signatures":[{"sig":"MEQCICpjHkSiBB2ZLWcYtMJLz4O8q1v3kw2MZ7/OyJ6lzeBBAiA5DeijWTQ/9QiH9d7Vf0UUQAG3SeJjQmBDEQOb1F7Emw==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168870,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfGap9CRA9TVsSAnZWagAAnBYP/je5Q2KYwvrA2NRvTYWK\n+TWSFTntQG700ADFrkiyUqR7YoHiOUIOYQOETOX0YUiaA/3aqAbjtEltYKSG\nsH7Jg05SdZOsJog1Nh85IYIQJBtclq/fStNI+AFDynCDSIsnJRbVveXIwYB+\nGrvmu3+iAHdqE49XUX1Sbunl0ZWGqUhOiLZVnW5nu480MzSHVo98m+E6uyxL\ngykfOozpaXi0cs8LO3kk1YlUtcsfm+LFHncrPOx9QpRTEdFXGrZ8VrHoJyRd\nzARBXaVVuuusiK9yV4Rtky/KiS35B9agaRSBmosqHXhmV+DWgDIak5Xn1DI4\nF2x0n4XF3xeS05mnbBXDe4627G5J57U/TO1o3CeTVLAV7FZvtc/qkVHlEo1a\nITywnzIjxwu15h6yO0KIIdh8CQntnRiSuvV/yeBlVE2fP9N4//cazl3dwYTS\ndopHWPuASLp+IjUcPMa5+sBkY5+7GxFXdegaoYvQchllXD80I2HKufI5fXqx\nVyvJ2ZLql2FRK4yeERNe6B5dIPvEtyKukn2wxCkSdaGj37VTSb+2RmvUiwWc\nMRYV/rigeJPvAcL3GNm/NZ0i8Z+OuOlSuTknEseQS+yVIdIyhQehZoK8oPa3\nuV+FDOcgLRp7M1NK5MQieGhiee++wJJupcxeWnZOj5mz4jcwqlJXv4xWJkuN\nmr6z\r\n=vKlj\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"027430af1709e46f61bdeaddab3ef7380faaccef","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-danger.027430a.5_1595517564762_0.6987826578261593","host":"s3://npm-registry-packages"}},"0.16.5-danger.c53e404.6":{"name":"@blitzjs/file-pipeline","version":"0.16.5-danger.c53e404.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-danger.c53e404.6","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"9dc2eeaadfcab0ca336b94d2dcef2625b29b62f2","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-danger.c53e404.6.tgz","fileCount":27,"integrity":"sha512-0pzUQ8+RgJ+PPQtxnnQT6iw8u4w0z1aVnsDbIJWYEYnHOI0bBCs2w+Nsi3kIuKH9TGo/lI50v5WaX7EDkK6GUw==","signatures":[{"sig":"MEUCIGRNZkTGDxA+x16s0YIjNCAoE35yv2oNJANFrfffI/g9AiEA2pAe1p25e32+zpib2qlBrYMEyO+qMnDiiAYgtB4uMbw=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168870,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfGaydCRA9TVsSAnZWagAAFXcP/0Zwfb+i+T8DObJdxAyD\nNWvUV+T4m+MSJIgQjz8mEUrO/4hglqaFQYeWiOqASG6UjqayHg5kR+HLintk\nGOvhGzqbyUi2HxodvR0IPxa0IlA1TZB0f0w+EUpza6L9g3gDESmdvuQOZnpS\nubysEh7Dzbx3GRXqoQsztPqJS/59U/gsoodfG4TwFei+D6IUTmD1wD5q4bqU\nCtVYlKtZxUrroEIHVqolwrxY2TWJUfE3qyuEP3NKChNNr5rY6czOj4j1xweP\n7KXfL7notb7UBpA2aXLbkUsISMrfTbp1+AVDy7JULRcHzW5rC5ymJRIJcT1I\nJ79tcmlzslDaFEJTDb0Cjm+EV9Dvg4GrI0QGzrey9bL89gi1/NfnREgqjpHC\nkalT3r4C6kfa1huLunymVLyDJIJiaW5uBQikfASxCBgMZO5OfD6+AJXzuby3\nZbVvRRAGfRntS1tjB38pQ7Ex0K6mYTBsPy4ktLYZas/7qh4OWUf4BBYV0Woz\n9pE/oFZx2l93QbOKrURq5Yy6lIrCZtLP96K1/XGmCf9ixjdlMNbDg8rURkGn\nY9YQ4TnjtWRwVu1eFkG2cRCx6mf+6Vuhk+7EAlB0ROtIQa3fiTNwqFMZIupm\nqwUirz2MlYX8SCpA8ZfFDVMck9jXVMwfjr/NW6IVv1Bh64bVhXstlaxGj5zX\n7ZxS\r\n=gg2v\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"c53e404e0274b9a1e96ee99181c4068f2505505a","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-danger.c53e404.6_1595518108764_0.67278595201685","host":"s3://npm-registry-packages"}},"0.16.5-canary.7":{"name":"@blitzjs/file-pipeline","version":"0.16.5-canary.7","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-canary.7","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"7cf0de1619a1cbacc957e560b85f3f75bd730224","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-canary.7.tgz","fileCount":27,"integrity":"sha512-0r2YvI4Wg7wpsOv4gd1oQpGopLQ+b4Z1Z30FoiWU59X4WEB1olC+xhBepF4GN7FUEYdSyuKXY5J0MD488d8B5g==","signatures":[{"sig":"MEQCIG7VMqBUfrIxR/C4ow8v1m/rsAmdm3x/V4LFtvd2+Z1wAiBG36EtNdPMpAlZzaEzq3gHEHiBzwUhPA776GG9lihdGA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfHByECRA9TVsSAnZWagAAhkUQAJjK/8cpR2xB47emm69i\npLLl5FdFjktcEc9fayYvHzXaps+rceIdNXdylJoRs4isLemEOYBUhPQel6Pa\ntOlZaA1C4t+BYrVliu9vutPuLC/Aw+QMDqf4ZmsARvCsuEb0wqow78Ijj5NX\nQSR3083B+WktckYiPq8mZtBEsHwcGrIZkaKIBcr+tEJHjxKI/H7KRIBtwvtZ\ng6rFWQeTj0J9H+jfg/CJriQ4T9OJZUnFXrFXza+H7BUL2/ZaUut/EDJUvUg2\nxzfPQRicNY4asY9dqlTdb+48H6oivCvNsK6+8ANn+J7UMvb6/6WIMDxUJnE0\nNWZJiBcEIvkJG7fh4/kvWIX9y8x8puL8XwYSvzbMlQ2LNeZz/eNh9ZeK3szV\nZC3oTGobnpp18iul8IN5usv8CO9QIdtSILSTkuKYSZySzbHk3WoACrtfn9Pu\nL1vc6vPl3tVKZFxGes3CXPGTWlY/iiDphAYowXx8R59O8kSAlnMoVNx1U2bg\nuP+VlR7j8aRpt/F7l7YWxjZMd7RffQPAzl1qrnJYpJVXUyDTzMuZbBePhjLM\n2RuWhGsQfRy994IEk+LAmaWIi286eM6j0dw0Ttc0iTIg4mPGoyCygJN1+Bh3\nSe6mgDnscYJ/WNvwZ4aBq+3DE4NGLVGcV6CWASHM1j8kR9ioRMB3z+dIzPDf\nwO4d\r\n=FMv6\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"ed233072674440619e57b704cb468534f11a7531","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.2+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.2","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-canary.7_1595677827912_0.3232240006265643","host":"s3://npm-registry-packages"}},"0.16.5-canary.8":{"name":"@blitzjs/file-pipeline","version":"0.16.5-canary.8","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-canary.8","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"34cf28919d3698f97c30af73bf411d51bb7a2dd9","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-canary.8.tgz","fileCount":27,"integrity":"sha512-tztEZfgGTnQtM1dS0OfZ2WWGL6XjlU+UHL3MuWivMyZdJ2RW8+OVfwX7n4YdUW42tpqgnAm2cybkkR2iSvBTvg==","signatures":[{"sig":"MEUCIQCZNFr7ODSbxKqUojmxb/PEtKGAdApKptpIQGFq/xrIegIgNowBFxd2jxPYz8kwNBuNIS1pKnmB/efhwSh1EFtLehM=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfJzDECRA9TVsSAnZWagAAoeUQAIplwInwMUUdnH1rnzuf\nD9BXEohVUnHlTLBJuXvfyzkwuLKHKUq5LGBm/9NWkp9WimFA8vT1jynvnXI0\nJnHS2lw/5Cm2stUX6dEwnG+qLqL9PVQKq2VEBMBDO2IVSHf1vVQHFYJjS4Ch\nnXWBPgjpzSIUs04N9cz9Z6TSLPGU379TPIHBzjkTMCHEQJZGXggRsUXnEiTO\nhxb1xdFv5w8t2PtSyR54lyde7wscvU6m/oTF4nRqnmhIsi+wWXTox9OdxZBw\nRH3+v4y977K5ee+3aMYWljlevUFZaRat3PrkCqARJTIHLuNuk/6jaehn3WTZ\nzG7hgTKgiWfqABT3a8o4z6+Z8WZnzxGEaqWqMaGFzg7EbsmxwLRqPrY2CpcY\nlpZR/LbpbKvoo670K1TMXcdik/yg9GrOtfuUEkTDCM4gfOVnIfpTxT1oay21\n3a2IymB+yF6tDLGgIpbyAttacMoGBlJpYazcBQidBB7ECfsnryY2JiI1/lFL\ndijq0I09jNfuIDFDuPFx59W51Oy18Kz3sE50Hl7BqhXayQlM5XhLaEAYQxg9\ngGN39Ikp569BlQM3nHPQL0UJZ+vPFqY7SmunsQhza8P6x/62ZJyGT2BSOY9H\nU+Ifgij+SEOBXCrCVM1rJKRcUicHpx1exVVx/RkVQqIng9J4VDAaZV2zFY4F\nyHtf\r\n=0okp\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"21896944f90ab0802f7f7e9631bfbced9e019e3d","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-canary.8_1596403908483_0.060051027937051416","host":"s3://npm-registry-packages"}},"0.16.5-danger.0ca895c.0":{"name":"@blitzjs/file-pipeline","version":"0.16.5-danger.0ca895c.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.16.5-danger.0ca895c.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"f325de584a3fc24ed4fad897a95c419bfc2ce1d1","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.16.5-danger.0ca895c.0.tgz","fileCount":27,"integrity":"sha512-zFBmMyS2qRXC4KRJmISMb9szbgtuHwqHp2OZo93cidkX1BdI7Zd0mNe3Kcd03vb4KlYXf8KkFF9lEcGziaHYXA==","signatures":[{"sig":"MEYCIQCjxd0sWjT9a3SR+5L14UYEk5xecgBbYJhYmHMV4Uo4BQIhAJefZabFfRD8AZAMlO/B/O88AwJtbHTyQup4XJOlk7Vs","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168870,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfJzWGCRA9TVsSAnZWagAAxuMQAIg52wsNf5DIeOcIB2zO\nQkbLGe8GmKvAiC3wI+mLAwHKe1zFs0B9yiz5sczMfJyrjlwnRchwPzQhe/sj\n+CgMQhn6VgrCmrYG1ZaixmXdGulycByomrRtBqbHmqikj1Jk5BsNbACGEFy3\nTgSgQn4EBHjD2LDqPkvNHM+miohAs0rTCcVxSK58dswPvrAqajgbVuIXf73K\ngngQGRxM8BZUZMUi5Ye+5zxug05I1yKHgiyUcrxdqLyidR6+yvrR/wSB3Rbg\nvfqMl7pDkT9jjB9LXyZ/169yg5zbFWGoz6yH/Bir/LuzhISNPxjOjLwLba0C\nsq7Nv5T2xNneIiBTeIP/qTytOtD8TcTBzHDyW37bWpbjins0n3E748RWGhCX\nC9TY4KmSFTBbFsgS7V2JoGxUpcRRv7SVnHgLQ4wZiXU4NP0xq230VHk6rvpl\ntPJjq78rGcZ+q35oUNtpLIFujibzYo4x6kRwgdPQpnsgtGgNZH0dga3w8pfz\nF1PzStrkxj6HMb8n8xVtSH+7haCXhWy3335ykaKPkyV6TwPOR43NZVjJwd8o\ncozSw39EAFKYxC2mmB013mauZbzLjbPIBVrNYq6F033sYE7oclWNFh+Sqq8b\nuyp9ZNhlqwkKeayioKxOr/Xeq+220y5d3Vvwc3+akpODx6jaxwwQPvUJR8ES\nBOoX\r\n=QCOW\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"0ca895c0c4a168cc0ab3a380fcabd844f82e42cb","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.16.5-danger.0ca895c.0_1596405126209_0.37467064974404884","host":"s3://npm-registry-packages"}},"0.17.0":{"name":"@blitzjs/file-pipeline","version":"0.17.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.17.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c5ec8d6568877420ec8010a3a5b643426fe653fa","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.17.0.tgz","fileCount":27,"integrity":"sha512-x1UW0qlpYkPF+xlkEFDTDwXLcPMkMCWGWkv0qEmhj5gFSd76BoejdMf07VUvh/f3usL7SLSeImgWHn2CzQjJkg==","signatures":[{"sig":"MEMCICIDwtc17IIyxgmTP/FJzTVsVcmUjETLYycqMUU5EdAMAh88DEtQmE5v1iN5Cf3wAEg/icOj4bnm6VMjJOyujVdp","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168845,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfJzlcCRA9TVsSAnZWagAABugP/iNjlyS6uaQ+SpQwpSzF\nd92vAKw1mTIj4ViF03yGGvCDVTYPQrYdwDQL30G1DZ7CbN2gHwd7rx7Jjo8I\nXeZxufK+LsvgWwQnIbgMaXzlZVBvnIgdTgcrKFyAOZWkydFYDo3yxiuRae94\nDLxMSGpJ5WLTdoGiY2IEmppUjKpLOt8iCjFwSkj/1l9jvrQBeXOhHdnudfLX\ncq/t0tnfdLPBxbbgNucKXQusSizUTnxuwrcHYIi35JMJLT6haH3YNs7zVJyR\n3OJtsaCth07PKy/st7eJudHPLO3LsQmCbbEWtxSpcee8Yq6m4jbTKQfwbTN0\n3CFdPo/znoWW5UxIkakej6udlVpkOL5+9OzVC21v+mErVVZlpcg/OUqwuDCs\nHzxPAJ1wU4rzhxTU/mj1yTmITW8I2ITlGiVqlu15aMSUdFS5j1G16kRk0wgP\ni5ejDHjVhYwqG5HIVCRdvKPyFp0UerkCrb2ACCBKMPXTGV6ffQ76Eied3Wn8\n9qqGXctLpFZxGwyz8jLWY2TYDyfp6SvNU/6wr4tz6YhoFdTWKsMC+ErWg2ys\nXhWk1TxIY92+IIOB5rmyPsCJopULQ+TjvTEq0ls5lxHUVjt/wAVW1GjYOzo8\n8YoNKuqusEr14esUb7SHM6FDWHr4geKedSmN6g6WoDr28jTRC7XMfkeeFrQp\neX69\r\n=5laY\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"740b57798f5623f710a84141bb6e904ccaf3d52a","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.17.0_1596406108449_0.8536896372338583","host":"s3://npm-registry-packages"}},"0.17.1-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.17.1-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.17.1-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"bb735eeaf3e8e2934e5faa1ee10b0a951632021f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.17.1-canary.0.tgz","fileCount":27,"integrity":"sha512-7QETJ0BnsPZIOmDVMcrd8s1BXRyWKVo24wNDQ5eqINWuOQ6x6m+AX7pOYFCe7pJdRJszAUEj2mAMoE6684Z8FQ==","signatures":[{"sig":"MEQCIFYc2ggU1UGR31R6Fj1yxgsFbQ0uNWWARHeBl/AhImScAiAtAUi6+XVoZTwJXlG8O/zcZh3a6A3cD8+27CoQfuW4MQ==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfKdI1CRA9TVsSAnZWagAAUyIQAKAtS+mglBQjZp/Y/EPT\nPSJRxn6dK2C0TbI5hXBqEA/kb4UXYvzAdKXNTfwpfIrvu0VPrMyyn42L20EH\nS9zEKL9EAyUTvFnnyxnd8MSplIIkwHbp/tJ3pmf5M9qvq7AM5t1Hsqk+YMQm\nMDcOAwCGLVjTZilzG7qNPo993hqbEHHXHK20N0D1oZt7iCGHbqjf1w4LhKNc\n9GkHpihAVJHmpPm1PubBvgfx+wE9RW9e/C7nH3PvB1O6Jr0si3EOyUMOQPHY\nJxPo3eVpmlVgQk6UFWIK012W5xcD9ndQPNEi5omHW0JIHjQXFfgQa7R3gXJ2\nw+o1kW8A3KIFDw1CFwZwsFj62I3Lh+G6TLiC9HfhPUfZoQ7pBka09U6WOh/5\nG1L0fs/G50NXtkHUE0Oio4K91WHnqvhNsB5lPeduZx7IbIAKjvjIPXICbagc\nnh3399jlrYHy43ukjMRFG0xjtrMpHY3c7ciz3BjxSP/fKz5UaIAkUQYgH1z1\nwZYtzqGAa5DF3tPpxYGIGQ1FaM2HEkeapkfSFL/yaOZv/yXeL9PdmEJ4+qb8\niKjvXdVm/cpdDMaDHBr29JS7BqOFYhv9I3OkjcIXiL3ZId+oxNTqbFRKllsL\nMx9U9IJDYKgqhVzKB8qE0wWJ54wbcy1Q8+/y3bupLa1/jRvDfK0cs1dGbu5F\n8JXF\r\n=UuUZ\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"5795eefaabf61ba2566a49c1ec5546cbfa1d7c91","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.17.1-canary.0_1596576309384_0.8799687574046886","host":"s3://npm-registry-packages"}},"0.17.1-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.17.1-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.17.1-canary.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"204b90caec7cd7acb1748dac4f5713c107f502c1","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.17.1-canary.1.tgz","fileCount":27,"integrity":"sha512-4gbe0zLvwwAUhy3g2fm5OqPUts0mTJSEdb32yLcoXvsdxXc+nB80OuLwnQeRqq+VJ8/2KNhUaFKs7gY0jkQ7Bw==","signatures":[{"sig":"MEUCIQDxkKQu2k8X7tHLSzCr0hgoFUrn9gynUQj6f3Wo48a0TwIgC4YlnFktzMbhBwTGAKZXiZ3/5vjcjQDO5tNHGCroAJ4=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfKdNRCRA9TVsSAnZWagAA+v4P/3TEi/AJAUynN6nCAwaD\nz2vooNh30O7XUs/1rUHQphNlfLxN40JlVvf17HnY4w6sB4sx3HqPkeYiWQZB\nI8LnDvpsy/zBOme96tdfRMZ/icDUTbT9CjXIrtEdi+HPLqY1fynrUCCCMHQj\n0QhVfCo7EZN6/e+P3z+u4YPdUzDtGiyfi+NPy1yISVTPdqzSJyaNeS6HDZwz\nBlmXSsV1+Nm2RTN50TR20525Jdfmw3AzSct23bEDYPuzTkrnkoUKRelGIdK3\ncQjo34TllYZxqtRFZjecsxXFnREia54j88xwVJiCbLgEshzgN1LtdmBqGMku\nP9cLDJkk6wjLS2Bf6OZhMhfC8fkLlX4ZJnF0OAeQozPH9fMRfmiY53jRTTVK\nvu4enRureAmipTwldY0eFVKszfvhy95F0uYrNPbDHE6kjORLT8ImxDqcLSRa\nGbGNfgCoflXvKnwSkszU20hHbYE4VSpCpy8jXFS/69G/A3BrRO34FkpRAfiL\nse/hjYgFXIklUTdBK1LaY2+bdDiUVpeVUWDHNv0DpEO646BrAbzQWIZt+4Dr\nnfG7gd0pQWoeY76kWq1ZpDxQHIgc1KMfV/CK3p4GQgi9ggueosAymOhs3Zpu\n/ty1ZoAhfE7mEWl/MTnny0/Wz4Dkqf5ElDBkGBjcBGmtPlqP0+zenzpZ6ODj\n9Y3Q\r\n=miNR\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"5320bc5211f6cd0941038cdeeb35478cb03169f2","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.17.1-canary.1_1596576593427_0.9486762319476547","host":"s3://npm-registry-packages"}},"0.17.1-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.17.1-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.17.1-canary.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c7bb568af4e640f2f4b18d254e4e488191b7b3c0","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.17.1-canary.2.tgz","fileCount":27,"integrity":"sha512-VPeS4wbmIs2mRjgVlDlSZDwZpuGyV3PmLR79rnFnPkxwGOmbgz/xVX7gkjLl1jzC8hYePgvVJFy0f6kCWqbyxw==","signatures":[{"sig":"MEUCIQDVWYDLO3XzjSlWkF5TsNcgcVUb+r6sKlKfLj+SpOKlUwIgI0lzjc0ywpTr5rRWyqk5k7UfW9xWVDXhx2RUJOWa3t8=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfKhJmCRA9TVsSAnZWagAAhkAQAI59yTKZE83zxPpp9nio\ndgn2mIEuEXnRsOS8z9m61PKZDKBEaYTADHDC6W8o2QhY+80Hlh+E39/sHt6l\n0GymPEN/BpaJcbbicbPFGI003xQuPGOr8sDsw4y9dswQC1KJfam/utUNzsnK\ndsfipf/VUaXh/y2OLy0a8v3UFrufXO2EGFV1bSrx9/lOaKtmprtDMn4QpByY\n3ap8QrHfg9/eEyca4BH1CWddmw8rUZr0udCqXGiXWmJRzL8p3OwzcGhgnGhx\ngfUe+wkXotOAKEVxNwowPM+Sy8sBXYgGAsMyw4Zrz95aXJBJneu9eBZ/08Yr\nP68L2y8pMV7Qd9tAEquIP0PAk9kmUsCMBEri/1XFhSgJ+n1P7EhgeJ7MwqvT\nwvf0h4d+TLkVlkvbCyoASvkpkJLpy7GovwnfQYe4aj0jlymYI8a17e8HFz4Y\nyuFmapmUAhNrsSla3mharE3giDu34Z6CIkFw5RiuZOlFqww8wo9TBvNUPuKQ\nTkXzIPz/mPc8fK+fHSwvtdipRMO8yU/xHIHHzHKKs1e9kVmlTQD2PBbBHujl\nUhtjmLufatwLYg0QB3fkNd9fAf4y8gKm96BfITV2PVKvg99Nzl7oi4z0cK9C\nHJNXzSfcbZSnkUxGMItLWiMz9JxGQNh4sJzm8AwtBDV9JFs8I46k4ay0efX4\nA4AY\r\n=+JTo\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"50a276f582b76d4d40cac146d32b1c707ee2126d","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.16.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.17.1-canary.2_1596592742273_0.6626176519856892","host":"s3://npm-registry-packages"}},"0.17.1-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.17.1-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.17.1-canary.3","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"07d0fe19db686b8ace11c90ed40f5aac6712727f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.17.1-canary.3.tgz","fileCount":27,"integrity":"sha512-c6+Q1OFbCHBqP2tBoUWOxg5epyDggu4fgjnO0MAFoPy3wMz5I5E4gAt+7RIY6ZP5zTIWb1ma5yPgHDvjXppIRA==","signatures":[{"sig":"MEQCIDZnUe+fhTAnUpPS9pRN28vGry/fnnz087Dp+05i7RUiAiB3qtSHuqJjtnaEFQFsGMKIcEn+xPbfWXrghmLmAkZ74Q==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfLypDCRA9TVsSAnZWagAANqcP/RA/aPdeRPOdPX4jwZmI\nwYVPV7aISHDuFzXLmkf4u85Cg+lFdhfuCqzFPG2trYC+poSZhm4yUgO/0iUi\noroSkj3VbDwnGgoydrAIiv9Ro+LkMGvlm/4keiFw49fYrvk8j5Ti7RRB01ju\nzOj93DyS7IqneC3g/DXaJmuoNmVvpEVbtasVn8G8atHGneWfkJ2IDwgf4kVB\nLP/dd3FgBdikuRjHsxS5tjiYB79g3/B5ppYbesx6UuqgS+8FTbMajDWnoZA5\naUuJ3Zi/M52q7JJ+WSos+oz2db0dVAcukpDhKn0ZIQlXBHTAVFC+lyF17oPF\n7wupRYVaSNXahC34kY426s7JGIB/I8ZS6wJcPl+njj4MjdjOUSq1F+yNEUSv\nxL7UZJfbLj7Irlhrh7k1Q29IVyI+W/uFy2a2m6ltxtEGItWUX+REgbJsl2VE\nC6NgHbWAkhdsFefky8l6oCLNr0pMJ7w1BN3q9Zdhl612wGm0RJZwWcY3dof0\n++gZrdZ5FwTytoeZ2O//WZwjpLtT8VojIdbdMxuzGuMjerYAQYqzN3Y0hfmO\nHMXSDBfEXg1n2OgPYpNERjCBVeO2WXyVG77Y7SvUwq5Ctk0O48/mhi1QYdre\n6UHaK5we4K6vRvThx549Gl/WjoOIgqF+2r4iHVkJVvAZmLHPJOaS7mYogPCf\nk4/Y\r\n=6DMr\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"9a3f1a1241a6560064d0ef7a443a6c71f9a93289","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.17.1-canary.3_1596926530962_0.3038014918100198","host":"s3://npm-registry-packages"}},"0.17.1-danger.518d9666.8":{"name":"@blitzjs/file-pipeline","version":"0.17.1-danger.518d9666.8","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.17.1-danger.518d9666.8","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"14dd182f3b8e8e6af90c789a9ebfcccb2cfd1e5e","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.17.1-danger.518d9666.8.tgz","fileCount":27,"integrity":"sha512-9njf+/vzW4BsbJ65TA4F/VXCtE2xXndw4Ox8UOzcEqo5Ej+MughWCCDolSTMEMpF1lFItAgDX78My7sbT85Lkw==","signatures":[{"sig":"MEYCIQD2HryBVnxGoHwZvKu4/DolebESp5/tXilkAQT1oGzRDAIhAKq607dvUGSmP8KQQSopsl54Gel1IrLXMVJBnWYwtxcG","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168872,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfNF7kCRA9TVsSAnZWagAACLAP/jEAJgsm1JGHdnRr9sRR\nbNhhwoj0vMFlp9w8mwRsfXAgbelZylZhNNa3QELbLkgCmoKSxLO7QqHULKrc\nzP7gzr8WqPiEZdAaJgsYE/YYpHHuAK4faGOHsQKdI8gm7hsMGjFBTLn64uEu\nDMwBlOaFajxqCA42JH8CqnG/6pgHF+M64BTCLkZE1YtSEZpT57pksZwNy0fz\nvUvvE0dmi2tiwoXtOw9eZKhpFQQsgFfRPuqkbIjF+SuHQAmzXJQ4s0PYpDu0\n2B60lm+vYxdDSXlDCiHcSRVkPp3tvQnRy2vgtHJRdtXMBY6r0c8r/ZU6tCnu\nLNR8Vv0/KQh0X3wy7hkZjjtaOizXHcWHF1fK+r5RdqmpBMujPwYzu5tK6Pha\nSU6cCiBu61fBlPVjVFYmZt8f5vprjrc2Knt+jvRrb4zzeqCqEP8EPjCMe7az\nHz7mIY6O/6TdC2cMXC/Dz87PsWkwTPApIVfbm/gJRdIrkGQ4FuD5rd3OfTOC\nTUNs1mumNUEBIDNGur8d11W/941P7f+p00gYB39fQXai4OSjhg1bT0EwHuzs\nP6U7RoMYUbj9OP0m+Y1eM/IIHo2MwydERRzJWEoGuium0SGYOnELO2qOlEzl\nAsJSwLj76IDCHkGNZQDmZKPQB3n6p/IEZRWWcET0hlCplNYbBtDge8NByqYE\nSf4G\r\n=GeAi\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"518d966608d406765a942d2ea0a4fbb2a0cb61b7","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.17.1-danger.518d9666.8_1597267683793_0.5400203061931383","host":"s3://npm-registry-packages"}},"0.17.1-danger.3b5aa900.11":{"name":"@blitzjs/file-pipeline","version":"0.17.1-danger.3b5aa900.11","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.17.1-danger.3b5aa900.11","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6d05cd52b9581c632f60591cc48e9903ba031ae0","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.17.1-danger.3b5aa900.11.tgz","fileCount":27,"integrity":"sha512-gqOhcqBcdHcA88mtUdEmLJfylLxMAKIwyOhKoUsRWPNV5RXDUW1hihuIdqB8HlJg+FJEnTHKQwu13tfSaNnn0Q==","signatures":[{"sig":"MEUCIDH3K3xPG3qrKL13F65ly9Ev1MJXtjn4lVrmpVyx2/lmAiEAxjERp7bqdUww5gTIvtROGThL/Hmpp/zg9KAQ6dGBa98=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168873,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfNGe/CRA9TVsSAnZWagAAr24P+QB3CYUNG9ZW3ZK/hKMQ\n4wlJBv4kIbwKqHdcJOvfzSkjxXosQlGFwOTaJ+zYKj/xXoxR8UqLp0xzDfFj\n0IkZTy7gH/k7FoVFmRoZteZZV67490VBXiwPWXuO9/KURwG63iVCChZUaLND\njlfI3IJjb+rfw/95av2NFZHtqD7ueHTCS2sFmzVzEhoER8/oVGvPLMx7SXGD\natIVm78UlJjAelepRzA2WHTjZXkZCJWl69XnffJ48H6d+IrjQ6/LcPCnc2ZC\nEETsB+ek+XHYT+i/FbDZ5eboTUK2RSC3Zsix8FypWaptsdL/Z4FeiQBiFb4P\nTFLU/9h03LAxYqHRrWIP7OJ++a/RnvuyAKQDA/0KxD+IIzuyi/vRJfrTz3K+\n4WMZAaPI7E286E5wUei1Xa4JfV7bbt279Lyh6pw2TMTqiuPq8XaWl/k89DBm\nKZ2iOu9m5zDWH7dq55w0Qni4BhX1u6mw7t/tpWjVwXvKXlR7ODWS0p6yWSbd\n6yfytgynPBSivSeA/Gb8uccNsqm8YsfdvMWezL/0Ju2hkPoPO5VBdOYnh7Xs\nOusbRnRtCp2l4WUwchw+bySIlUzOjpq887XGE108+2n5kUPbVtJXfvLgoyf1\n+Tbc+S6oeCR8uaVqYOTxf8kaMEaaKssFWt9u5fMXVHtAjwr/x3ZwiHmAUGQ3\nn+px\r\n=vOPO\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"3b5aa9006dc8cdaf6e0437c6d34868b04bf30372","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.17.1-danger.3b5aa900.11_1597269951139_0.5912450738820318","host":"s3://npm-registry-packages"}},"0.17.1-canary.4":{"name":"@blitzjs/file-pipeline","version":"0.17.1-canary.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.17.1-canary.4","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"61b021b15700be8ae4bab905919b4e89f888dac5","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.17.1-canary.4.tgz","fileCount":27,"integrity":"sha512-Y6fUoIUb1tLtR9wfNJdIIHoCR3FLm6UQ7jy4nX6sOPQX4JjsqkS5pJqUMJidiP9oh23B3Iv4qOhoa6JXvboZjQ==","signatures":[{"sig":"MEYCIQDI+D1RLf0A8gPOsJvR+OnDu0NBw/Nk46/i3quuGSvmfgIhAJeR+KW5pmG0iqmalUh5BvvX/LF9+rl8DkryFR6fTwCC","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfNHnOCRA9TVsSAnZWagAAPXUQAJm6qg0TBqzmRuyuPHot\n2Nx0RFJiQanUC866act/gisMZdDfGHXgxaRFSSYtRvrcEtWy/Mry+w4PhthM\nhanVASt2vlS+md8EarQHg2F67FUKtdjov6ffJl+vMF0Xfa1rYzQYzt/fI5qg\nX4+m/UEar/LAlflD4UoFASV7i+Xc0OtZj+eBEblSIyuxJ+8GWq66YR+C2vFo\niF3i04ZwE4pVBwRQmD3oISLU9iPptg4EFp6jX1hjjiyJ8mgRnoWfXWFeHeL6\ncqsZT6cziHh0k6ulaLUMob/zan48Rvszi5WBSulgR7CZZR21Q6CTACZq66uD\n9jdC0H8D5QYhmYPuk1Jk9QxTCFP+fFygZWZfhK/lj3YwFE5Huxvs6W0M0FJa\n5JMxQ3tS3kZAbajMSiRp3k4m0XML5WwWc0h07Qhlre8koiEcP5xL2bfS7n2t\nxz8X0Obws+hijWu33gsySheuTqXPithWVkuE90cogKOjorZDYgWykxudcYTz\nrH1Cv46vT/EMQVNen4WOekdfaupZ4Ejf3dNvNTOv5GmAnzLDe4qTcVbki3Or\nWRPIgIDJhs2845MMFO5F283/CDBqcakw8kyC4XgHY/A6BUzzGo75Mgo3Faly\nPy8EgQSwtKPqvFqsgKW4Tuqd2bbzFOBu0/GD0pwUS9T7Wnri2tvZBO+Ut3M/\n5pRo\r\n=nyvL\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"ba0aeae3e53861b7155a7e1ec30032dff058f81a","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.17.1-canary.4_1597274574161_0.8481812542430687","host":"s3://npm-registry-packages"}},"0.17.1-canary.5":{"name":"@blitzjs/file-pipeline","version":"0.17.1-canary.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.17.1-canary.5","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"5d81848bf9e7d75dd7d7db06572ca61ef4972463","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.17.1-canary.5.tgz","fileCount":27,"integrity":"sha512-zlTMbgqRNYVxrGU6DDZCv8iNuUXEOeiNMZg2l7vkMKWCworpNRenVxG4Uyq/Aekhb31dyeiPAjP9kxrDR0KqLA==","signatures":[{"sig":"MEUCID9SnXKSlvFB5eTTx3fCqOjWALhCzp1nVhVC5oeHpporAiEA/QKwu9S6N02czry+2rZvy5Xa8OLGi+d+HawPu1IcMgY=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfO0DDCRA9TVsSAnZWagAAaUcP/3GhRUc1cvzHBJEvpqcH\nw0JxWgIVg3B0P2Ftn+ej2pVTuzdnqc+UbFmz5lkeVkoMYmfi5guVfsBDsZpX\nvwabOc417fPGI569HnivpDFzCLez+1GFfvnDeUEwltKKyAZ7d1EOyzjLM98q\nbPBJG7v9nwpOprW2zeaKzAjmKK8XzgkChaXXONJDyrrC4ZsiHZK6OlFpiP6W\nxGi3RTLYpum/cj4ty3H72r6uDxlMyK05aoJSSmoOraCtBuDZAi54YCQlKvar\nClJby6sHDKrBcBEPJQr1Tbz0JeaLx5gRo6JKr+8lEtXdsjvJ4dgmfukfvbng\nDn6bWcsE6qqWOnmdyJ+iSVNWNJMfRe2mdrSh2l+gCMPP63l5KOWr+CR7oRAA\nGMckSVDtiDdtl2PxiLBzUpsLh0luLRgk/Eck0ml8wFFtSzAx40Q+/aCjYORD\n5ZXljsmyENv+APymYCdTwTXPA/H9QXvw3rW0FUzJase/90o96OIfbvE1APut\n8i+SaRVv36YZGJYQ1Iko4TCdyB+dVxuPq6qicgWXp0dql1X5YCe4CX+lkQNU\nhSfb+Y8BmOV+MoKumZ56XSIuwXQtRW4vrAGkrp7scCn2tae4/yK91Axk4LX1\nVsG19xY/cL5W5LCeRfay7I7UrURY2Tx40IDI2uHsWIrOOoJ3hzIgTaWv3lkp\nJ/Kd\r\n=QuWy\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"b0d61dc4f1addfee98404dd0ee1560cf5de8e865","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.0","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.17.1-canary.5_1597718722674_0.8054074476599922","host":"s3://npm-registry-packages"}},"0.17.1-canary.6":{"name":"@blitzjs/file-pipeline","version":"0.17.1-canary.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.17.1-canary.6","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"a8ef3c53f7a539b8b16a3cd30fc284a3d06fb62c","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.17.1-canary.6.tgz","fileCount":27,"integrity":"sha512-YNc2HQBu/Mid2yhYxoFlffv/QCHAevkhq51ilNVaOaKdk4nOqFwuRGD4XL/Dd8wyMJosvTS6iDhLkLsyoj+Ktw==","signatures":[{"sig":"MEUCIEsDJNcTkoWgX1Dfdhfn/9FvGAjeN8rCYkbfXO7O7Z3iAiEA+Rio4lAq/RiilLSwx2V5ANI9a+xErxOcgwJc1DJPPGk=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfPHu3CRA9TVsSAnZWagAALssP/RSCiFE5rVvy3Te82ITn\nwLbkOnSG5GVnD72nmHeadOCvC1nCvRT6FL0/BcI7DUeBSK6FAKoneT30VJ7p\nsPjhmXAr92bIYsg88srRZcbAmLlnYI7gogE+NtVVPem+eQ7JujfxUomA0/+M\nh3a2b+HcGvocOEa22GFghdkYVggor2/56GpcOnr8COcYnXb9VRFp7uzoDPPS\n5eAsedd7kJDo9j9CS1rFC0TLlzvId2VLQ6hGR2wZE3G+Q//0g+u1Q5Z/2WYa\nRJj3URLHp/07HJadSjTJREQZz4UKyf13xxGXkZFKHZVpomGfVzXcp9+4y74i\njs2/5cdhwIsdzIXlOrmFZ4WaTgAt8NBqkd6IGpGCAYkm49Yh5MR9Dfs+AFia\nnhX92GIumu4t9NyCm4dzaNYPqKSfcPpw49eBWoDlKjQR/ssLprZDe+zGd2oM\n/D0AE5jwiflo3kDos9gYpcaLys2cg0DkjkiabSiPmjxYHcXz+hyCUeO7BsFj\ns9Lxysk1PkpXyYU0pEZoemg+DJibv8Mrfet4aRFcgoHdafhGIU9dni1mWNpb\nVAfKnuGoAexcnmbMyB3yott75EIg/jJzQSJ8L5cxPFGzwvIC/7IRum/f1sWS\njrosoMFohwm4Esj6/3H8ln8Q6ITq+n14B8LmGfcsdWcsIN1dz1Y4vbKw3dPF\ndLA5\r\n=/71S\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"370486291a57a740497e405e80a01f3df251bf22","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.17.1-canary.6_1597799351011_0.08420603871998855","host":"s3://npm-registry-packages"}},"0.17.1-canary.7":{"name":"@blitzjs/file-pipeline","version":"0.17.1-canary.7","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.17.1-canary.7","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"65804757cd64ce3fd90ab2b75216656f2ca315b9","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.17.1-canary.7.tgz","fileCount":27,"integrity":"sha512-8JGIJ0l/di+OQM00wO8EH5Z5RLUMHtSQqX71FWLC1IqI+hoqVjlUKVlO+tfj0LULsPwGR41kQwb6n8lcVcIJww==","signatures":[{"sig":"MEQCIC3QqufyCqrR+mQGJzjVTKtm28g5HHn+4B+kt7mJvMIoAiAfJCKeV89YToQIUkn8/yXpBlOKShqyEnm5QFM9eILLRA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168854,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfPZtJCRA9TVsSAnZWagAAAsgP/2UnHf6ApJoZeGFVBGnl\nIBEXyqO4ecnvpXDnAZmCS2kV5HEn0x4/qBAc47sagcAxFZGp4Do6dKFcclfL\nkCFIe5lhCyqxkSMtAbjmMU1VsoTP0TSOAlZxF8wVLv8ywoRpvVq2pSexPIm8\n/haA0fygfUgfr7Bkp23CvJ2dbelflKYN3q9Lao+omhChj3zJgR64WQxKI024\nqU9Hf1/q/BEnh5Y+XkF27XjchH2Iks8x9+k8RTd6ao8bnppItFunowOXvRG5\ntJJbQ7F8zoKaoBSL9UgSoFsyx/8pjMJ9MSFs69ByzcjMB0hWLluxGAHO4C2x\nXVcAq0NHEltgtFLl9mvwdiYmh2OXuFscJ/xCKJp/JO2nSUpInMBSHXqKXO4j\nlpdDmfpL/zRrfbQsPOaBi9+BLcYeFpUyZ/13wN9g+TPyj+NFDUTo/ldoaCFj\nrHvwHwfKmM6P2wZYY0bpmJqUln5qfsDG3RL0j4nyag7wlVWhXWUpevJceNBL\nKAF7qkPRfFyVEvXzFNJGOowf9pzFC2a53OiwtUVFqDWYXF3sUnFFuT/IXJZv\nKWb1H/Kx9/qMEhrr0DiLqEUcA8quAqN2ckT4wFWDPsz8JPtF9U8R7SnbozHA\nKPY+8+AM0l3lwIKAOIOGGFSMWa02GpaJDAuNFTRGV7L/UydbhSo4aMcHL25C\nsM4c\r\n=/uGB\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"5cc54ab981b158e3d26b6bcb7827bbdfba638105","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.17.1-canary.7_1597872968974_0.8300912517545767","host":"s3://npm-registry-packages"}},"0.18.0":{"name":"@blitzjs/file-pipeline","version":"0.18.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.18.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"e26d8383981e2b99fafec670f919caf9373a5650","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.18.0.tgz","fileCount":27,"integrity":"sha512-BAKnK5oBk5v+M3qKG+6fcwj7E8NEeUlUUMw4NkGEvggqXbWO6A08Qd1+8Yl9rIs54QDo8gs162Ww4itSsPZCUQ==","signatures":[{"sig":"MEUCIGgJtolwWW4u9bDoqpBHRA0fWry9W4rErk0jM5Knams9AiEArR3OeDHSmsd75Qpb/6emd8PNHu4SaNDegex8oWOM8jM=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168845,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfPcadCRA9TVsSAnZWagAAnrYP/0euIl8EVAltRw94pu8e\ndP3OqI9N7DYsy8FuGROR/i+95oyfOX5KyfIHDw1K7DUuvq9BwALHiVdFNomI\nMSi9S5STsQg1l/xBERaroLh671pReT7UrqKN4f7Aj/kLjnA9FfDoEGHKnxnC\ndqo7iCvGTYYMma/NF8etHfymXNBkvweW2+r+wQICkWQ/YrBRW1aEXPbV1n+3\nJiJw0RRVVg8r6Roo7vXtmqhcKo6EdaehBk4mYvLegFfn2gHSKnWavfCDx2BF\n64Ye9DUXICmulqB8krGfNWsrnexGE+OaHCXOWSGX2OQ0ABHS8G/DmxDxguda\nFQP4tz4r3nF75dWXMHOEtIFxREAdtqx0SNZ5ou8Kn+UQdGeRH2Gv5JiApeQp\nGRA16EctAqzj6gQnvEcNjkfrUYgRk/UNpk0tDvOoqq+qN6HLa0CRIDt4bmiU\naKgBm8cgEx7V3X6711zmND5c60gMixZ1HKPyuuNthctwjnF1GmGbqAAaQDTf\nxftgeMU0SxKSn74k21YzbxJRG+MIFr0fdMEM515YOP/1u+6+aBRfmLQuNtmf\ntuymUaweU/5oF1uY4se9Jy11R1asdTsg9dhtPlcZcLY279d7jV6i6m/YMKuA\ndczlv3I9RHpmFkLgfYwVF1IzYdGlysGHVzUcFz37r0GU3km53zaPfyGKAWx7\n9OBd\r\n=kIjz\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"a6878b35d0ca37cef4809603e81fb522e5969d71","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.20.2/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.0","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.18.0_1597884061214_0.04175655834687242","host":"s3://npm-registry-packages"}},"0.18.1-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.18.1-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.18.1-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"8ff5f244e4a6da466973aaf7c9e857347890c982","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.18.1-canary.0.tgz","fileCount":27,"integrity":"sha512-oE1aJK35W5Uwc0b55CLH0gcvo3yKnOl96G9HwrO5jYFH9ysx94bwjsPM5L4DMjR0DGlNL5Izbf0bCuLRNHarKQ==","signatures":[{"sig":"MEQCIG+k7X7aVmcFbro6BndK9MrwcV6xX9XWUWwh7wsvkmADAiB5oZGqPIwOEM0NDSx1of1hMhLtOl1y15gEP2+lWpeRwg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":169733,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfS/7VCRA9TVsSAnZWagAARLYP+wYPLtX0V8yaKl2rtDEd\nK4sZcfV6o2IR0Pba0uw3EOaKk1TXeGofpGGkmPu25HXxGTTkAKyU7B2u2Xk5\nZAa0RzKFWXdpiS/vA3HDniXOnHWI6xkJq3WJ3iNsn2L8O4C3Y4IJVdPTGm4k\nvEw1Dt7pPdeQMsKDmmHBOOAPCM2kUpmyO00FonUTdf+4h/YIhk//SmFFE1Bp\nThfc8ulkOUaE7wnpDsyDljrDm807TmpEUvPCF19duuqgmx0IubUAWjZYaqdq\nArWisYL8Qgidpdpm/NSPnO4wSfL//Jk8JHul7xyd4SpboSHH/Uy+D+Y+LKHN\na7mmnUzp3Blykvsw9wuCzKntohFDxcmBgr6JSzhrg67qOq0g15362wioQj6b\nnftd6uKhAVfjaN7DZYhnQ7MdI7uPBHdCAPoSU7hpzool9Y4sn1lVuIHsM40i\ncESt79x5sykhovpxzRhi8uNvhVEDHQmcMETJfOPXgefHK34A/4kBlCe9s+xD\ntZhZ0KHgeou5QGTZ4BghMZRjGv1ARULKqy59xmZxDx7WvtZLWd/JMdEelEEk\np2vIml+J7C8JN80UExN4fV88NViuwmU5Rpj0e+RCiihgf6vBCyvMbC0oNi7H\nL8yPCrbvektvq+u3KxUQSc4HOfFK/392GBGtax2o2tcUd7rVFYfS5/mCrE9z\nxcTa\r\n=P+30\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"b21df113a8fd8a607859d243f0f03d42818ce834","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.18.1-canary.0_1598815956651_0.985990747220588","host":"s3://npm-registry-packages"}},"0.18.1-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.18.1-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.18.1-canary.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"5ca0c7448c60a5f9f80ad5312dc6b89da70a398f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.18.1-canary.1.tgz","fileCount":27,"integrity":"sha512-ri+LtolPWRjq29sm71sL49LJ92/jDArmtM21dvrI/IUBUsVr36j57X3J9jUD9aRCC9R9kjjroU79GY/yaKarHg==","signatures":[{"sig":"MEYCIQDPBZgO76/C1HoRarQJSoyQFJgghVeH6W1TlGu3LKeKUQIhAOWeDXF1nKjUOEFXF7lzz2Znr+nwePvxUsWibRKhWdKr","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":169733,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfTAE6CRA9TVsSAnZWagAANRcP/iN3tE/ttLFzBcaMKw/m\nz/DrD/YBS4EWdV2jCjXWzeswV7KXXNU8DkidIXQSqj9pH57oG05HSazETFvI\ns4V2vqTaTbk40euddVR/7BQN3dtPunYMXowibnl2KXXjxP454XhOPtLirAxF\nPtKKSqUb6WbPl93ZBjKm+T+OJbLw+5LJRxxB9UpIYFn8wgvB4zgBy7tgBjlp\nHuQj/y9CLYgda8iPciHb+BglCO48OkVyuss1eIDeYBqlZV7ao4B5D83Vw04+\nYZV7lkQGKVllVoZnqeB/mWAK0OXKYz8XRVbU53siT20XkpKbEe8NPU1yuAYc\nSTcOAHngtROF/hlpY7fNoTZAytl4uirAvve8fWM3gp453Ey9RS3QqhecEvCv\nKEL7pKkvIwV0y3V0WR5hbT0kJIM2AmIwCuXw2RQuWaoPuh6EDHOsJ5rLv8b7\nMmLNjcuE8HqgplmYuCmoZegzJafQS20+VnBHljcmtwkxK3kuxHa6iUjixCCD\nLQQtIMM5OhdgZrItBZCaWvR6M9MwQBapB8f+KDeRUSf9emuow1RmuVTVo6TI\nhB6aHN3ESMiGBIucisauKxIL1kEWFPysIGIz07Gsok6M/5ppOZ1m6fuvC6IJ\nG3enfqaWJkVDBr+swFWFzNAaoDoRVUOJhrmwIuIDKQsljtzBPvNYMmZDg0S1\n1yco\r\n=jgJx\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"b5d564fab8ffa2e29e61539f2d1f13d67b37426f","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.18.1-canary.1_1598816569804_0.48238001474816583","host":"s3://npm-registry-packages"}},"0.19.0":{"name":"@blitzjs/file-pipeline","version":"0.19.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.19.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"99856b60fcb49eac4b1f3ee7f2814bd303d49579","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.19.0.tgz","fileCount":27,"integrity":"sha512-8hJScs9wAzpY4RdlHwMRQXkHduNpLko4awBbfX4Xv3JAuxvyBp9IStH1cK2T5pPVU+SHHfFC1/OSHlT6leQpXw==","signatures":[{"sig":"MEUCIDt3+C567y5BIJ9MxCk1wgr7+7J4VkFgHSFt3qD9YTukAiEAypHADmMTYNlHAVa6LNs7iKiJLyLC6d7f/1pefEQM7CM=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":169724,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfTAYMCRA9TVsSAnZWagAAw2YP/2FNAMmPDtqKo7fu2kKJ\nZpuq6SJ037acvK+xlZj7IAYIiJXmixJKCvSBubYf5kriRwdyrzlnS90mM8sJ\nXqV4HInLJ+V1tqURaI6znXN/wWUWEWzgTj+8DVztSdfRSJgJbrM+2+iJVV4+\nKkJRQSWLTsAgQ7Y/4bkmrMdfiYhMMHALjVnVQe2mqqsjNuA5cRUhWXzg5ic/\ns5d/i2FWPefSsHDDTSAvNNTNLEguuAdD9ipFO2tnJ08BgC01rZjxJ/39Xl2J\no60MvddntV8zBX5yF0m0V5HI+sd1hi4w8vkzwUDYDWYe6K/pdjRU27ZUlFnq\nAqq+29KxlUsANzyy70mEfgyk/UgSujn1QkfSyFAnCyjTeMXrxxHiLM7uGTPl\nGvpPZXy8ZBg4U+nRmxTlNyPj0hDDho9cW/bW0yvyDahkLsrFIybhfVcP7NLJ\nJXK8M60CZHL81szWDtXR+w/ahZreyDIMNsKrk5et572PaQK+UptwqQDK1a8D\nBdFOrENSflCW+PqZ8+iynLaER2ZMG3gqtFyjN5wzkUOgYVALBO3GxKZdRELA\nIxXEylhpD3RWnlRPlNA9oteOnISVGJbvaGlW9vLG6suoGwD9rXjC3BGZqqss\n7xwVjI9pMTabe75h6sRO3pI9SsIWMhWDcKvYOIPYdeOhdfzQ1Km/M19kbhSw\nIlnA\r\n=psge\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"a00a52938769e439d6287c9cd0fb7555ac8aa4df","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.19.0_1598817804526_0.12681721124794954","host":"s3://npm-registry-packages"}},"0.20.0":{"name":"@blitzjs/file-pipeline","version":"0.20.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.20.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"8f51e514b2a88739a430b145bcc6cd13edadf18e","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.20.0.tgz","fileCount":27,"integrity":"sha512-Ax9zlySRGDNUjej1TiRxw9kBDhUNcbyGnjLuicQWdpIDGnmmquW2QlAoVJ9qO6SdKjZ/A6jlAqiYUSrq4s2WIw==","signatures":[{"sig":"MEQCIHJNSZywB59wu6JMoLiCSaV//8UWJzuP8T5p9LT+tEesAiB1T6gwSfu2Ly/1jMmYHmpyHz+41ckRDkkon9XIenUqYg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":169724,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfTVw4CRA9TVsSAnZWagAAb6QP/i8JRWUHbp6YW2h1RGHc\nNKgDxnmD829R7PUA2+XR/ACGWkR2s2jc9V4hJqKhfAKrgxhTtcrLZakCt3BE\nkKHGNT+tLHI21AkwVZi0yBrTDfSrbrMHXBdX7NTLmJOqpMdCz4Fu+RBYaw2Y\n9cnOIwlVlAkpYRfkuWtshoOpPEyDsT2mlqiGG1xLIF5YhwSieOdTOr7LKLNg\nMY2YkSTPSKUWBXRizJyq1fAODv2X1Hzs3wQm1wwQApZjA4sqmDmCjLu5BNlj\nfFfXBDXFhp3w+VTU6zICA6bk/W4vUkfljPF9vvQGLrrfa3SVGmC4RSmctyMl\nP+HcFaWdMvyXY0xdW03/UxfMGAI9SEueQKgGSlCmNkeUYaQ7dXgKneyPh2xC\nMRve2tH8MDXzRF0hwgwqpyLAj6wM2hTgl4NZAOm9BPWtcj0xNYMeddst1irW\n1yCNAWL3egG1+W4G/qOOip/s4n9td4YBaPctC7RxdhPBn58EgZ+GqYXnRw/c\nceZ8b3LF5ng5KEWgpDHu74JUHc3+m7dHXrB5w/Jbs0XuCEwGwd1/ALT4xXIP\nI1awxIEt5400pEVtTGj0MtfcNCYIjbWRaxsmYWVoOGaN3eP9ORaaGgguIFr3\nZrVR7qA5pnyBbuG7CJsJE8oZBrxpLK79b1s+KRUoxEj5Tmi3mUNGpB7qrbLE\nXqct\r\n=olOA\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"1ecaa5ea76ee90430bf276d4a00939183b14026d","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.20.0_1598905400098_0.3641485735202459","host":"s3://npm-registry-packages"}},"0.21.0":{"name":"@blitzjs/file-pipeline","version":"0.21.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.21.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"b09be9daea71baf060719456c7204aacfe6504bd","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.21.0.tgz","fileCount":27,"integrity":"sha512-TBQwlCcMmmqthizZH84g//b5kf0syUwQtrlwB7PeJjfvveXno8+KFhq9qVvZ0573CQR3u7fwX4yvQnEgYZ9ukQ==","signatures":[{"sig":"MEYCIQDyjF2A7D+r7hU8q9Tk0Zh7fTocnu8dD4l+bC85DAj8vgIhAIUQertB9dR9r+tUKNHb+RsoKnBxjMaw4qSMWi3xXckR","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":169724,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfTq2ACRA9TVsSAnZWagAAm2UP/1m9h82cJlTRe/iGaKMc\nMIaKCZEoNHN36xeBXEyOlzS06OamzjnV4sTBTziPQ8CFTO2MRyGpaxck+2Ca\nBb/X2tleTwe6vSqYWvz2EPdsrbGEzTAT9KWbwvfLPX7Jcvu33li2hERdYXXZ\nD4FQN4abMLSKE+znjmW7SeEd9MBm5g3jBx9JGhHapxFOYF1DzjkvJhak6him\nq028j/GfjnXg+vZKp1rq+eq6WkgrLM6JcIYM9jUO/PBrKQ2IG2x35oFTAkdu\nDIrpJ4swAu1zNjDuOfAIXYZzfupRhuiL1nAgcTXL2YVsiNeZPdtyhNTn9E3a\nw+L4/ALoEPMmwEULXJ5Q5SK9oC9kEcEB3ZKm2PFxtuBLnDlg21Cw9uZNI320\n9l6bjFvN+dAKF7sry4TeCHuzrtuU6qL4KLI2uJvvsKMQSUQpkCYM+AJ2OzIr\ns7seEFuUyR7GpeRagWNH6NnT4aLGMp9vmWAbcUZZwtdXM02uzpyBM6Di75jG\nM+cSXeF1oANaMNg/lbwanzkeIL/9DUS4Z6OPCN2Hi1DbZchTYjkYaZgnipNR\ngs/jr3gdjs/FE/W1rxF07CpE73igG8xIk4dsLWcyyQJw7bcjv/Jtj4rkcq8F\nXtRWfY3fNOmEpRT8ux7E2dr774HB+QNiwC+csldtRiggB9wvKppsz1dGkvTR\nX0am\r\n=SM/V\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"017c1ff813530a4720b238d7b3284c33462b61e3","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.21.0_1598991743889_0.11586114805233327","host":"s3://npm-registry-packages"}},"0.21.1":{"name":"@blitzjs/file-pipeline","version":"0.21.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.21.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"84ff21589213d27a20f4bb770ef50d6755deb8a7","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.21.1.tgz","fileCount":27,"integrity":"sha512-xbhJbd+uuMukBkZ38tHdDQwCx/u+39QZMWmArcHXP/W07tQiNpn5RVtbqnJrvb3UsjC1piLtwbbhy898UTOfxg==","signatures":[{"sig":"MEQCICmGZtKstLI0Nr69Peb9URrkhAyeRrQiikdVR/9JxfGRAiBnZRSp/zvjKxdtkX0LtJQDvICzdCw9jRySn77EqbHxeg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":169724,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfT9USCRA9TVsSAnZWagAAoVIP/3fBsNJsAMpdMJ2/+bI5\nILEoVXzQCqps+tSZNccKjN1ZGJ89czFDTqc7iCfr4XtDkUaKUM+aPUASZMlu\ntPm41pPrWDmF63bAx26r4ib2jbdtBaNOR6qCdaLgs5x5li+wuFwHCXCxjzxS\nLr2HvTtRwZ+5Do2o101m/cCUdUWx3blUyYKzLKO+PEHfwApqiWPQyRi9Y0sM\nunwa1B8oRrv2ybw0j0WwyFc2+k/WqYr5udT+xqPAgglGRwKHtQYx8u1VHQ0m\nRHvL3lOWp9ThYTfp3TRbSL8LsfNReI0wSvp8JhM1KUIuzZQZyFNOUpv+lwLf\nni48nLoxOdcPy3U8m/HpPoAhrqD7Q73xVw+v3yNAkEAjiyc2SWoJCxfdB7eq\n9mY9XPu9P5/N/LUfrMC6ZSWRBDnDvTPUGWcqWedgfEv1GGAJDiZo7iUgo832\nhfa5+cFPGN8JndLa+h/VuCXO2G5sFF0dCBPuczJ09OyNy7ceYCXj6j/BENPr\n90XXh7Wg5yDHOkrG3cfKsfgk5RyfGcC6wO0lh3FbidEcIS3g+2PPp9nM9t32\nU+G+wERydLmKuP0zSrT462ptUj7tziqkKpvUgGlpvHZ4soNvCWYndz5d7FRh\n/Psewc5yYlW7+NVOsdxCIouMK88UoVMk4LGiqGekXF4pL04FtH7+TZsMC5DL\nnL8C\r\n=WDSZ\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"ec0273f09d4b41a0d403b5543a0d054c6fc2b467","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.21.1_1599067410308_0.5249985326811977","host":"s3://npm-registry-packages"}},"0.21.2-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.21.2-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.21.2-canary.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"a91502dbffb27166b1d421bd0d05bec427d96989","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.21.2-canary.0.tgz","fileCount":27,"integrity":"sha512-4AslVrLDnW7EYvuk1DNVX1ENDwSiMHzzWUzxaf6PN2xjtZSqnjTVV1Eo85+XTszHTuEq4vBxgSnVFvOhB64GNw==","signatures":[{"sig":"MEQCIG7Dgib9aVANmHKfAWirqkU9FfcN9bK0J8Aiki1WQADGAiBLV5GQ+yhzrmcSrWxYjY8a+UrgaffQDiaL+SFd9ec0Og==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":169700,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfWOFECRA9TVsSAnZWagAACEgQAI3m2nApMzmM7e4wbYM0\nllwLQbxUGWE3Xx/xX84e5Z9mwaUvzpAU3F6fC3LaSxQxrTUhVSYrPwYI7TBa\nXDRAaz1m9fy6GG0hKDhzTYrBHnWRMOUz8UNpgYFNScevvsN0+eiGc7+uja+m\nSEREYB8hItY4WvE8u6yFjNNPNIQA24Rd8Ql5/fQpEunEzMIqYL3ruWcF4LAc\nsWNNKNmHoyh/U5GC+KM+d5Iggu/6/7FwmCum+Bul6L0oy4mp/IetuYsJxDAB\nl0cuo00U8YZArQLFrlepIfZ7PcFy5lCtZtnqkRuFoBQN7IW1DPbGIRXEzw4U\niayqpzl5CYDkKF5YgftOCfmhBjqxx+EcG+Z9duvyKJ/3EBn1ovUncbm4NDDZ\nAZjtwRAvklyyQO/T4RitHshFaqzYMXk7gHbzvjNyV/PUsv/NBoVpGOq3PRlU\nXeHtR4xVXtM29jTxidu5f1pjVLQbqBy17eE+i+c83o85SBGnWJCZmlIGeTAM\nyrJCa4mfkJhZfXz4zbQjWmdzvRfOcQLsC/ecTn26VwVIaYwCs2bFO9zTDoCa\nSY8vfhVKVuWdJ9ds+jwojX2yzDznE0SDJGVGBTsi/NtypYUnAtKjYAKhKFS1\ng9G2HWkjjenNuFX73MmfASU068GrBGAvWDajtiMCuRzxO1H55ubsW20QZGn4\nndXU\r\n=IvoE\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"294be124f27abc242d38b0fad5a523f070058222","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.21.2-canary.0_1599660356396_0.7786642063944702","host":"s3://npm-registry-packages"}},"0.21.2-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.21.2-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.21.2-canary.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"10207248a41730adcae6829b5750cdaedb0092be","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.21.2-canary.1.tgz","fileCount":27,"integrity":"sha512-JYIr1MHT48QwoCR4hqKzXl6gusyq1aO9cFq4NYzg7fWirYfeYzX0bPbk8G11XFP9XFg2iDOuQH267lveivzMTg==","signatures":[{"sig":"MEUCIQC13kTuza+WDeBmFy7s5TggZwdD3Kcbkni62JNxhsZkagIgIpSLH/KlA8HZB+6iqK7hT1ZWzUcqKDgShlm7A5Ujg0U=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":169700,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfWQMiCRA9TVsSAnZWagAA54cP/1z6xbOBIayhSC5WWJXi\nZ38rz9Kw6Ngh5fK1cuaVvoFomrhf/j47V4Mt9vbKDMplnF15iFa2FbP1oAlO\ndEdlqw9BmVbUGs+hBSGUfe71IO/RWIHwYKy430Qo9Rb0NjbE3d2MTMXPfOSE\nTwQozFAQOBLi2K3N/wwzdWhgUJWOYR6t+ySZRV8ygQzqeZ/wOcHhQsYhxuBW\n8pEaa6PJ6Cjm7daja8+mwidFB9FUHq+2RBZMQfGrTC9kMJJQrHLhFzhW3+vF\nf+XGrR2p3nh7ZQebwMSMFFROIaSIBJC/Mj/ECRBgMnF8AsGD10ogvBGO1HI1\nCJEnmJ75dy2L9Fp1t3aDWuZ9eMwAkpXnC9qEM18WbPnQ6YaOH49XEn0j1cza\nwSwZec0O0Rek+eZnLviXSkfzdKalutsYjmcblvxaaHEES9XiEG0RIZH/q2E2\nWNpkOf14t/CocztqfCf5uRDb1uMjKZeB8LCFcHhLi444gZciBMlo/jpWo8EP\n/tlAkTNtWS03udElh9jVurKZPSCkBZkZuNAzhKPGeRS/teZNjszdM9fV0p4e\nvirq+9I8IGGY87ASz9jS6Oi6nany4ZQ/qKjoxp0PiVCDUnUpiNFlztaGgWcV\nSpKu4ixqnt7vCpVU08Ch9TFJYDye3rEKpVBJ1yA4pPAxytB9mI+yIXui/Ag5\nTC0a\r\n=rQgb\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"46035af2b33bba5286ed8a950b367dfa561b602f","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.21.2-canary.1_1599669026357_0.9115703605592891","host":"s3://npm-registry-packages"}},"0.21.2-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.21.2-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.21.2-canary.2","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"518371920d0a68d7d78d7dfb9a7bf2f412b377a3","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.21.2-canary.2.tgz","fileCount":27,"integrity":"sha512-t4HtnGoPejlsrQqhgpNgnJ7BpyJMxwVvTbticQkVQUg61VDHt5IW6kjoH+dWlGnZbwuloRpcl168UeOGYE3u6A==","signatures":[{"sig":"MEUCIC47sz8P2BTDnXEg4Y0Oyz26JM39jERBKfat9y7SHBIKAiEA0Umu5U045d8psNyEALp3rwkDo6tZleqZ0cKZ79g3/fM=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":169700,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfW5NxCRA9TVsSAnZWagAAOkYP/jrLlBLJTXr2SJ2RIqwn\njLzA7Di3RJli4Juj9f9vSprJ6aSTEbtiEG0428Zj4em8a3iHtXiIn8bKoQd9\nc5loDlrEzoEnEzU3fjjZkGFFj47BeSjJEMF8C1X8Ar7Rem4xP5JzhSvhmxyh\nxBL6Mq8pnohsqjVbQ7W8EUfghfng8QkRVIm7VFzn56P51sKMKZuhOrl/ZcMI\ndRnKAgUhRUqYqupO6AcwtFEJHWi2QatkaIyHqg/xw2kAjMgSHfXUb5FrAMec\nxsp2vSRWJ3WQJ93m1o5F9GxAmegDzJ0Ho6WYDo1eN3iuL+6e81EMRE80Ed18\nQooHqXUG0nwNhgRHo0Gj4WdeN1nCSjYgsSee56JBQcJI9XsbW/jNKfQCtxoB\ntxhEDk3nUrO8gV6v/ejGhRXunpI4+V9EmWjii8CMfmF1oSWdBzEQ3FHyWp0N\n+//WHpOHQn8SO7txI2mNowAToU3eYert9cwQEWozZsuwuCsq087fpectryXs\nHsvVIG+ueTxosfzKu/ZFgCFdAGSP2isP6W8MvT2FPOhzPPj9Dvs+3S2Nc1PC\nUALKRd80eQvJ3M/qTWezDLXdwGoMJ9n0UG3zI0wslRjhuYNDQO5HhJetkfTB\nTTBoQjBRjx7Mz+T5xQ0Hu1CgIgKVEqNE6lYIRUgjXCvk4LQR26XBrwUTTXJM\nUXj7\r\n=Sf34\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"bf2734d90701c73d15cfc67541728c4b26566049","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.21.2-canary.2_1599837040794_0.1262211132165474","host":"s3://npm-registry-packages"}},"0.22.0":{"name":"@blitzjs/file-pipeline","version":"0.22.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.22.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"25b6617c8434592bcc32501fae3bd29e1d7ae8ee","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.22.0.tgz","fileCount":27,"integrity":"sha512-QgeHpQMvJ792hQocA0qIIHHWH2i28B9wndmZroEYW2tbI8QYHYFANlQ1S3+YNQJVuD7mIp7JzC9w6Dd9PBLCtg==","signatures":[{"sig":"MEUCIQCv0C11AJEMu9ycdEktGWj5nwLuGfsx2EkU1fSIa1ImhwIgeZ5u4m27AABqKk2Be7m+Qa2vctM/CMCZV4tLPk8WQPM=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168778,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfW/C6CRA9TVsSAnZWagAAMj8P/jb2I1xAWjjkhOW5ZUCU\n/2owrAnhZs5AYv9+CdcsktF/a+m4kk1NWUKsbCLl0bdkGdxN1/MZWcDOQvw1\nZO1S4S94XABfbo4xAQwmAg926TaW0OET92TeMDmePW2CtZSv6JXldzOrlUkA\n33gj6himqmH17I1wFiWP3rOvk0RUmOeR7ZwJkJq7YFa+s5xJ8yQm1h0keqsi\nBMyKuTKFzHDf0Gqv/wuxaXzIWvZ5j5WJn0IHFksI7X/RG8YR+jOPgAGOtosi\nHE1VG9Hu2MMBgmXYzw2tg9Rkw/4FBRHYe0Q1UwmcAhZEif9JFYPYEldqmfXa\nVTWJ5tSqoRK91TX1bWFNp1dtxgjfAK0vxq5yck4fUTrTQ2SBETwOoJ6GJBP0\ntMr4pKStqz11ex+RjJQ7LVJrQwcjVrTTiHR0Ld+dawiW6souVu1C+7aFyPpw\nqCYR/HJmDpF8Igi4qMlMPHZP11vEYNkS9gZZBRIJ98YxBSp4r7xX3c3YpJUw\nPKGgOOlnfLJMxIgmTGpIOAi9n/FdpueozFhvC987CdLHNaZ46Pap26WpWVNN\naeOnEexVXLIK++oaRl6M3PGOXfeliGmntGu3sN0j21ANpcuDdRdsptugzDnX\nMmhVMCM53vR+KgqhYItiBH3/FgXW1hs+39Se7TlL0g6sSBxnReXahDeAqIcb\nVFc5\r\n=grXy\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"ab4670c21bdf86142750e434b6d54e014dc40927","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.22.0_1599860921920_0.6137351993432336","host":"s3://npm-registry-packages"}},"0.22.1":{"name":"@blitzjs/file-pipeline","version":"0.22.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.22.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"324ad9cbe0289073a331f9687e968fe3176a2e80","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.22.1.tgz","fileCount":27,"integrity":"sha512-gmfS3SWHYfMuRRfmuu8ZRWcoNYuBzK3Fv2FY/IhFhSSyRnDJD4nNBLvwAKQyoDODSTVwgQPbAO+D1RyQ8fkZlw==","signatures":[{"sig":"MEYCIQCR/HlP9xPhZGXI2Z09yPM4jmJtHUsPanRKc7da72Xv4QIhAPReVBVsh349Jidey/RmgxSnMzcpqCjWZDw0z3c6YEKe","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168778,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfXQ1nCRA9TVsSAnZWagAAuc8P/1fbhuc8uzsWmCoUApHm\n5FuLaKV/74tPg2jBJuIkioBC8a6LzpqqkF5fxiSLKP6++LOSppAGH6pTU9HL\nhULPFq6YwhvkNebkx5eGWcupOiRSDaoj4oJNkzZvQmvbKTjXur5GSrqz5EBL\nTv8Z0VRrzfayvXU1iGWyrbE4f0YISpd7TURUZc5bne7d7JZEDrcKok29ZsAw\nSrqtOLVplRYQWkdP+p7hCausdNAcuPgkJxz7Xfqts+m30mapJN4Cgbl3YA1N\nPlwQpJ6KA5I7Zn2/HSUA2oj92RtBfyeMcnnyR6jptkcOjBDG+pnWpRYmFl7x\nBIjhwJEXFs0ir1nOsgX+5dTE9BLOaxXDptL9BThpoLR360slRIc1BCNO+sGp\nJjplpBwFUifjvquwzSeQ7dn2kHNwhbct/RHCiBSXm8mfkgfvTDMBRc7UXChI\nuHj1IJhMmiTYe4Lnp8Z/hKd2IJo+BhBHbkCWOU7sRWakyK7RffTQtKU4ISBz\noa7cNVr///zlFXPiA7TIjv2ZM/OpXi0Wbsa1KBhe/M7wkj/Lwkb2rZ9qZABE\n4lDI/OMNZcLxZetv/pPTbY+4Sd2i74JLIJycU8YJ37seBosM5zs0HXTWFn5v\nT9RjzvWAdT5O3AuUVOrTDlnKUsu3BffnSImxc3VxxJgJhwKW3GwcCvkX01qz\nF1A+\r\n=2xEL\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"d953ef795ab9a7adcf238c8420d8644364233ea9","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.22.1_1599933798964_0.24792369603980635","host":"s3://npm-registry-packages"}},"0.22.2-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.22.2-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.22.2-canary.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"d0268f039bc2308325bf2a1929cf2c0295413e73","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.22.2-canary.0.tgz","fileCount":27,"integrity":"sha512-+ZaoqQtEO+2WciY9a8pkeOZU0I+dkRgKNeNY45WWJuoF7Je9JoNb9AlWAGD8hauh/8CM/i7oXUQooss8jLbNcQ==","signatures":[{"sig":"MEUCIDqjj9fUULAUw1DbjFWKW+/8NWbGbByomTMwhFDDjiwfAiEAh4ekY3Iss8eT+HdDl++wyKwmIu7v0uHa+8dB5wyfoiU=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168787,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfYCHrCRA9TVsSAnZWagAABMoP/2x/zAD86VMLyxfYnEGy\ni+4YPhZabdPMXxBgWlhyb4e9+VghPDqcPm4ISSeda2lD4giCZ+N1Z237Q3qS\n5gYFbiV+5ffdU64+/dmdMzVqrvPpvZZLa/svE2KLD2zoFW8Y78kdEdfeLhTu\nDz6JoQY4xHmEf+71QR7qu0UFuyONrmEbTnAIfv1YLsFvYwbdzl98m+h6OvWf\nf+69B5MbnepTGuDkHER0QFsI6lXQ8+8kgYJluf0G2vKeafcY6r1lZP4OzlZ/\nB0lnhcSm9fSYZmSIex1IMoH4VnYRausM/P121mmM6XknYBvRxvv5BtJ8vmg4\nD6G1ApMcDVScddgTFqx4wUOqgmhuoA8fXiDjTC/zujm1XG6EH3syaFLyyGe1\nOQ3wcWNumBOSzWitfJhScD2PymmqjVVU6qpw+GDxlreouPFTIRbvVaXxLP3u\nFEp93ePiKBhP4OO6GRPrNdWtN4Vi9+7atiAB4Jl5n00Hp0mTyxCEf6RYqooT\nVXWRq2/btIFtSVFvjFBCHTAadtp2BZOQBWTV89Ue80f4vkyV31oj4wEoU2QI\nUahWfJNghj9jrZ/02HqnvEM3aAJ+iQz02Xd9wIw8Mk+K8U+71SGNHB+/YsrQ\n0jnDCmEKgXno7FudEkw8HMzqCS++xYiDU/wIDLH4byFDXyCKuGiW4GPNGP7I\neokx\r\n=DzpX\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"1ae2bb3ee337c2b9468231119ad959780efb30f3","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.22.2-canary.0_1600135659449_0.3103438089022692","host":"s3://npm-registry-packages"}},"0.22.3-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.22.3-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.22.3-canary.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c9b9736f93db748838321755aea033952b452c4b","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.22.3-canary.0.tgz","fileCount":27,"integrity":"sha512-xmirN25MukDOBE+hmdx4d19fyf52wJD7ytU81p6kz5DCkO5+MgC4juuegSv3w6z0IitmvFlNFwUnoPy5D8V83Q==","signatures":[{"sig":"MEYCIQDTW30noSdXMLGz+vGP7RidiBZ0V5AJc64EdDgh9V7uLwIhAJreGY7JBlBn91hIjLiPtHNPaXG9TlBYqyBeYUamkhhE","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168787,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfZrgaCRA9TVsSAnZWagAAb1gP/1u053rBZPUuWXnB8ebB\nVrd5uL+RtQXXwF6phsKcGaITTRajn3FQZA+6BAHAM1lIdwzjXrUTV6Gx2+bc\nuxg6OWSTNjCGzftbcXf5nJ/TmINIbkfV3HuZ51Irpoax1LXlBA64vpthqveC\nUXDWGg8+IAooUOHAiCJqavOU60WUdJU+Q9PpeYYqyRcs14BeGK4iS/OtS/r3\nSqtsMrsO11xcv0raqLQb5Gpf14qixGGNa7DpCVMm+oX6Vyh98DtsnBQoBPVK\nGHLr2aGyGblKApLtH5qMimfeD7NuzkQ6duYcCuO4slAteTtEa+JrTQk9yfAs\nTVNfCOwTLzdkJwLiAekaQibiaBOZ/8XiJlOzdJ8Jjnudm6rri67sOQoPP6Yr\navuzSypO4Jz3BUkcgNRJ0Hc8MVvNu6mIfRaDFzzY2uFOZCmmvoAUTM5y2R9F\nh7XRoWJhUgpz5zGSrhdLe34C0eOw51y7JkbeCRren9UcZ7SEOYvXWPT4/mYH\nVY5My+vDwNi85e9MxtlPiemBCc7rG96DD/x32sUOkUzWEPwp/fZ0zedg58fO\n7qOrJn+OVI67kLgtByxXrj9L8LgX3Q7kDZt5KzCocY+qmyiTuEnnjdkZ4KF9\nEOscQq2jBOy03pBESNCWIz08IdjJ9E+tBPNkejgNJR0hGJrUznB9HNn5Qp3l\n03Zl\r\n=gRtB\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"fb44203510d1eb04f343330a244c5e4c86c8deb7","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.22.3-canary.0_1600567322290_0.8760564533715594","host":"s3://npm-registry-packages"}},"0.23.0":{"name":"@blitzjs/file-pipeline","version":"0.23.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"0326f015edb77a01c797c6bbc176fd5ec426b35e","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.0.tgz","fileCount":27,"integrity":"sha512-lJZILpsxkdMO25mGrodYbBf8inLy6gzwUn0QfZhEBrA+pmmmPZjs4se5/UwxW6RFksJHaML09Zbatgls2LYO2A==","signatures":[{"sig":"MEUCIQDpCbIqLJTYd2udWSDzwBAmfyNXBsxTK8xOIfNVlGvEYAIgN4IqoRUImLchX7soG9xkwleWw0iTKWNHI9PGhcEBq2k=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168778,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfaoi4CRA9TVsSAnZWagAAZmwQAKR33yR/A877EfpRYHl5\nhukaw8PrgfhsAXRcMVzfdDdCj/SxDXaX8/4B6D7eUMiru9sICJm1iVbWbQle\nHTJy67H3zbZqcJINmPdESY6GJIjkwsb6toh+sxu9vMUqUbLd3/wFv/hXkLvu\ntjx0JDEKvpk2E+2DpzpNdZZveYZ307Z1JffsU+z+4r5R7dc6SVEHTcqa8gRq\n4gGj5hGV1x6OlY7/pxkgXW3LWza7b3Zqg5gQ+aC7QtoWz4u2xu8EPHWT0Inc\n3IiEtZ+oyicqZcLVTA5g9SY+Dx6B7I0QQIt2vozA4/O8ohw0cChGJ5y2QdkA\nAIndKEeKR8hrK8/fzkVppNw3p8dUCP5ZmbhL4pHr2I7b4M1SKIY5fYqwz1nD\na9T0EqkjAG/0Z/PexGe1WpzVrL8y5BnhgFwloamWOEI53HKGVrmZxf2d9Wa3\nDIpdef8clbaTr3NMIRI97tfsOs8FJ/ehFp7UfRGYHN8EgXUCptZ3YCuVAefS\niMGBFFtgrSSD/9ev0vh1c0ii7Er4yNBoYVzX9DGVaZ8ZIuXUlejUlm9+auVo\nJXG/BeAqh/KiCCIP3Y9Y7wOEcfYTjw/0EunuVXJj3vHiSFc6gVSml9dWpayy\nLtfgIqdPKIoTmFuzHnfzEy8yLyqNUrhfN1wxrWZzrwdke+V4ovNyF4M8lQYv\n91pm\r\n=9RNz\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"9e88b114960167a8b2e9ae664d0b943a7b342658","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.0_1600817336474_0.16986213465776245","host":"s3://npm-registry-packages"}},"0.23.1-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.23.1-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.1-canary.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c4626044620ec0c0687d62227fbd56a494493f24","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.1-canary.0.tgz","fileCount":27,"integrity":"sha512-PlyhymrN8TMU8tZmONcfUvwS0K9qJpWmGdMCpCp/+IszjTcgHPKIx7bPa7Wfi4Bz2cn3zxEReBqioB+u84l0Fg==","signatures":[{"sig":"MEYCIQCM98SyLq1SCvo+NcLtwLql9bCv/SbKHZprbEg0xZpqoAIhAIvEFM9iE9VM2u1CFoUupwUJai1m9yrnqwuSCKe3RmfX","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":185663,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfbAP3CRA9TVsSAnZWagAAZJ0P/1xESgDPC9YVL1y1iTVs\nce1RlftRKPRv1EgBTxgMhnmQOTl6qw1LBmSuF+aIetJiPlc1YEBC5QovRwfH\nHRMHUvdV1Hf6t9cHD9Z18YploW5wbi2gkElzRodpgu+ltl/c0Fzyhm8HHc2h\n+4qHJPSz+BM4WDbT4zw5caFh91JSg1YWvec6YoTuYW7l7bfuTn3NdhpKIf1v\nu6OlxVeV90gy+v+gOt6ddfOAgVgbn5mfds+kg+SY4EkhjEE9vOZswPSISXeD\nQKC/MEliLMTZgulSPGc4pwtCJHz1BeJIFVNw+iN9bIomiXHfRrlci6fopAQ0\nndEsP4r4iyDqpemhkUKv9MwQKWplRvqN+JSU+5rk4xVYQj4dJ33IqMx0PU2f\nnkEnhaWiwFOQXU1K6NzkYzYIgYat+r3tXwr5UaMd0LK841r4rLioeioDeiph\nelOhzNOwdWNc7dPlYspaYVGc7zK9yeVmULEar67pgRAlTKeJDS0mkMymSFB+\nxwvXjK96XImdOVk3HNCXWVFz33hrdfB8uE/fvl34Wp0J1hVjMGUDwJXbZn7o\nkNM5BjIpMGh0s0HtefrFKRGHpm/sglR96DEvqR31/hW+PYAH9XWfLPZS9ngi\nXknkdjRE1d82Vc0f2/9Bmg84JH8ooaH8nOssmuGSx3XWJDN7hV3oiq+5lYxa\ndl4l\r\n=vtct\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"d623d8cc33a83516719e17173ae2c0bf0685537f","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.1-canary.0_1600914422670_0.9890880094973087","host":"s3://npm-registry-packages"}},"0.23.2-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.23.2-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.2-canary.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"38eed932efaa559f38d4bc90416d4812928c6686","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.2-canary.0.tgz","fileCount":27,"integrity":"sha512-aK380gtvdp8tEat1JMZx3nwsXk27hncYZNUpSrCF1O9nA6cghexAeU+o2XnM5IT7s6yjw4COjGikLlNTT9cjQg==","signatures":[{"sig":"MEQCIFRW1ZkdQ0or1wtF/joEb6rBkFFYTN6W4zMX3Ir0p+PdAiA9BAIjwi73AuIh2Vq2ux4SqQuwfM+TgX2l2CZJiWKtSg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":185663,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfb/cjCRA9TVsSAnZWagAAST4P/3Y+ib9p64373FfcI8yr\n4AvTR0R+40Qni9Qb634ZUUS75Ws7X9glpTn7ebsEYR+6aS/zjexjB/q0Gl9g\nD+W9O5zS/etMPPEBYCxSf2aj31+kRBgb/VddsBOdCUemjMV4F9c+7uMlibHP\n5gc8V5BgVnPagcDRYN0oVsQDPuSRG6DKtUBSrIkTvWlVsH81ps5jWKO0V6KQ\ngRh8h2fpSkNgdJG+plpCvOWorhBulh0JdRNoxJOJpgNSi6wNOu/xgTFYzxgY\nW3jCUujwxXsdkHXg10OT6KSEJ/odr+pskZlNqhi2E+yZqPBHYXMTdvFbpzCd\njquQEHs+O62pTOiqAUkvNtlf8rfKbVDTfTtvNqNkLBFI1808FzcFXzL9TjQ1\ngaTXgP4dBPGGpLZ9STBUL+ggXjv+JBP7qpgfepfY2LM59WYHVN3rIq2HxImB\n82TLoA0Nkx1vEYUZJysXLvXDtl+uiQIBwlWMYYdROYDtB/+vRv61KUnQpnON\n6gMVlBnZXEl4Ycp+JCG93Ktgm5qoTf6YFE4VmyH0d2iUw0T/1hAVcX95lrpl\n3MRRW38BD1DQUTcHCViMWMJrLnEDEI6NUbBkRK7VLIsL3j5RJB0xeaA4C4Kw\nBIHfBxx6TsyOX0V/w9+1z121brooQSd31qAzYfCuRja8WdxjEuZ0hZhUFcXW\nxOKG\r\n=supC\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"08abc334944adefa9bd95ab49d6420e5c864b70a","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.18.3+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.18.3","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.2-canary.0_1601173283427_0.4722164708352796","host":"s3://npm-registry-packages"}},"0.23.1":{"name":"@blitzjs/file-pipeline","version":"0.23.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"00130580d5140bd844a57d607db9bd7300c0256e","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.1.tgz","fileCount":27,"integrity":"sha512-hpseT7F0AwgeHvFRmYNggsyVzMypHAqXSwAg3fIrlnimqiBszJvG5KtPUIcy7AOtxq3bniKG+yiWE5BuXBYAwg==","signatures":[{"sig":"MEYCIQD64bpAUPi7JWWq9McXpxoSOl9TC/zHkRJbwBnxSvGbgwIhAM9mPu+LEcb+t+BvSVgPnDNhGXG+lN/9iRqk3bYrJDMN","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168778,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfclOGCRA9TVsSAnZWagAAcxIP/jQwP0PZ04iRqXlnMj20\nq5TM0m0fm3zPRhE7V0XMzhGU7PLZfQeaG7ITPgIO97eI87MwhwpaLZFTLerG\nVzJtT0mFWIeSS9PX4wHCJ094NBel4URRwHs2oa3S9T7fMl/HVEPJql5Yci/j\nRq6joijSSQCOpZ7NVbUIbnWroE06hrSeccF3jHouoClaBRCAMR0Xzb0hGLq6\nV2xJEBSt8ltMzUrS4dY1bBWeSnRZ1d1syaT+3qI97MuziZRMKMo5pNf8llfA\nbrB2yWD0OUnf7pfVYO9GGmU4RMepBvycPzHhzYugxVRnIZTBr+ERthPzWgmT\n12cbQkEAKjeQ06TyzidBbt48zf+yrbR1m6cklfT6EqxnIuKo+SwFz1B8FAmM\nZGUwgZCfsTuyC3vu6LL2hHHsUopeJk9dB3zeLeyywOYye6k2YgViOzOlACn3\nWz4cFs7gBV8p+YYQ12Fj4ZVPSD5aTk+/E6XQgjfzB8lz6jaTEqkdJ+vakqB/\nEKJrEX44mKxTgEoprAw+oZbgQvSpWKgKaSsPc9pPT8yY2H/AC6zFr2mH5x9P\nmHy0ewpg2jre3dSsqY2fN5VjlXRm+2vkfiD457ba9NVxIYexMTNkOZLEK3NS\n/S6HaB4IR570B0srnU9fEG2flwRf7nmUx75XpP/3Vyty01SOlBiz+zT/sW7d\nY6Do\r\n=YZYn\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"f79a0cdb4c7d5a5b67565706ffdcea4252e7ea70","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.1_1601328006160_0.8910924005722547","host":"s3://npm-registry-packages"}},"0.23.2":{"name":"@blitzjs/file-pipeline","version":"0.23.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.2","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"92c9ffd4c36a0ad3e785e0b09d8661e7259f79f3","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.2.tgz","fileCount":27,"integrity":"sha512-qYa86f5JSfC7sfW/0fDeow6yiwn1K5fvmLB02xXbCq7oyfE5UBKEUgYY+v7KShYXnmqEfHLl6Rv7YEkEUp+GoQ==","signatures":[{"sig":"MEUCIGKmbcYkaC1tdyMc/Mfb1LWOPhXULM0aNrwFc6JUU9NxAiEA9IQh5PiMQPf2zHIH8NaVVyko97SM2fpbKcu0nzA/dRI=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168778,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfclUmCRA9TVsSAnZWagAACW0P/1eKs+WXZBocA1DZtQ16\n5zAR/b7l0Csj40g63WvQees71e8KJ8EyBZwsEg3/YAKpbsxXrwjszv9d1fk+\nqmiNYnVQfLfNKDXHAMRaf4mTy7Fy37P8TbgoCqVEVd8vCXc7aHIrcSB1iKVR\n5aSrXakEIMOWqhOgN0XLp0n7gLYFDOcd5Vkrg0zMguOmlhrohtaMZrRGf6Yp\nGFNmR9JnrkhjhVhPP1ufUJeVCzvUe3748Btj6185vekdDgfReL2CNfvA9lmV\na6MRA+vjQR+a8aTOwLpTa8KF8T88NXy7MV+NjwGxlMkVcZ4R5yiCqv0id3zz\nyxF6mcUpYf+ok3Z89gYC0v/poiaTmPcDkKQF0EsZfbah7KMzePqbVhV2UUBD\nBzoz8KMIO7OQSTxMN5H7OZtQs/rkXodHsW0jpbfW68XhNKH81bw68F4Z+P8Y\n5PwwwunbmItbsgk7HMaWzk6vRFRrJH2FvI6Gti53roFi+hQp4pE+hBeS+a3u\nZTyYe8ASK5qa7k65k4X3U6j29Rt2fFAKFo690vE9Klj6I/wJVLlEo6+Qj5aK\ngv14CUtrMhhg2bu/u9H2dv2u7DS/DAl5D3fRNABe0CZxbzy7NCTfG+fbdV9f\n8b+2RPKpWpP2e/ZRFGPrS6BNJfXqdKkvGvm5UJ9LQKa3j6MZF4LK5uhUnauG\nG3/7\r\n=eYZr\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"15a349e3a4265ab98b0c41248d8012586e1e80f1","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.2_1601328421889_0.7411141817127425","host":"s3://npm-registry-packages"}},"0.23.2-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.23.2-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.2-canary.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"afb39427eca4a6416e22cf6cfd4d1a7e9e0baa2a","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.2-canary.1.tgz","fileCount":27,"integrity":"sha512-v/oRHPZJtZtNKHYBI1XK+ZI4ygQvaDdbNpqXQynOPIhUYMwi53dPIPBZDyeYKurN2wtkdSXgb/M5AjnslO6oMg==","signatures":[{"sig":"MEUCIH2LRddweUQAM/7slz2fO21U4YRfu+AQLkUGB6AHv3cpAiEAyw/rCkTJTC0NwwpGnw4hMELxNNp1iB6RGGLQwAFJOVs=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186059,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfclZxCRA9TVsSAnZWagAAKJYP/iUjMzITOmOZRTNZkLgk\noTZ1pVjAKBtZgF5eLYkadMQwpIpqQs1fl49PveZTsuIwWB0wulpLLGREqjf4\ncTNoAJcjBzATp6jMDMv+sxOec3mFe5XM41hesf6M5pQxyYNmdw/x6PGTeIyJ\nA/xQ9RYOJx3aZ+Er6wlWBpC5didsSTZMKrPHX8mmWWhypMKIXzYFTONXakKL\nsOB9ZEBQmrcJTESERP5iBSBIISqYcu70zzaUGiOtj0leFDueU2bYpxoass2u\n2+LQVlwwQKPdFBEOEDcBvYmzF7gNG8/bLnWoijG3TIAYeJ20oW3/BQLfFDIG\nSJnsvOiQ6G/5wI73G0bSlr23/RTiWdCqQWKOhpfOaewadRKYqmlEl2VrdCZx\nKhgqoC451M0QKv43fUI37rbH5lv5OZEmNzA9rfNq8lEExwu/oohSMVT53NcP\ndZcCiorWa5y8AQmYMNnf6ImTnJwWLRDhczH8LYdpG+Rckpoi8uGCoIA22RID\nslFoHZJ0mOaTDTjaOn1x01YtIPYM03DOnyQkhiWHJLOBAJeWSAOihHk4G8BQ\n8dLyCAgDKC41zIIdnqy6xq2zXHkcWLic8+Rz/7D9qA49lRzb6obUKGB88vhP\nyHgSD5bbI91jR5GSE0ojed4zv6rz/OghkCW0lmIVaD4gFxI6Cc0TC5IQcc4B\ne3Wg\r\n=6Jlo\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"2b0fe98cf5bfda0539809827e4fe2fc2d4501836","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.2-canary.1_1601328752883_0.6719640580823163","host":"s3://npm-registry-packages"}},"0.23.2-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.23.2-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.2-canary.2","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"8ad48e1f56cd9199e77b0646394cb9f6c54d8bcb","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.2-canary.2.tgz","fileCount":27,"integrity":"sha512-yQSekWmkT29MPKwVB765+LQpcjvqwVbxdHTu8QUM1YtkQ2rRi/w58mVQRl5I3pfaY18hfLvAiPOYZt7MNklLpA==","signatures":[{"sig":"MEYCIQCJ45xSiwXFS9V7IU2UjGDj4BOKF2HP63f4PWsH2jAKSQIhALkHfbcM1vRdrLiP/rX+OWopdDzQ3naV6ld7kB5R5YDv","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186059,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfcmqeCRA9TVsSAnZWagAAEd4QAJPJokUn/vwNKnBZHqO2\n+n6Mdb7nAD4x9k7Pw/c5zWlI3yy0QN7YtYioJ7qeX4Z/O/1jv/xfXmgx9dlz\nmKTnyOU4HcE3XJiDTj1iczbQ5/ZraHNPUICHxXm+KZhWlFEiCyCXkojNtPV1\nLbkZcg+ehBpdKv7lff4Z8okr0Wh0EBOpxcBlEt4QNERbtZwew0GsKqfz0MBv\ntQmnkGtNW3wgPseAYTHBuRmKsrQY5E9v5OJG/k8tJuWWUHAAyCnTWMDS8pGX\ndftJCetL53rroj77A8atHCYIV2aYu8DIJ/PPVkR28ZTglWWBCT6QRmf7+JQz\nIACxLpWwjb9A3VZBsdXtizIu33QNT4CNJq4uxCDi2b0OgET3op72BxAG55wi\nVZATNr6jtsc64TT5OyviTO4DoAE/+D6KaHFilejKzzEPw93AVvCHU7A1JC8n\nTeADJYR64Gti6WaplNRb0Vhzz9R4WHUFmw28ndMU1d4NYbdRj76nkowHt6Cg\n4oDGcuHfewwmKMoxTFY0kgSLq5GzFOEZR+Wc/xmHdYfsqtLQ345EMv2r0ahi\nkfJ0hLR3X4k4sufYvAQOtZavrqQLMlgpODSG1UUbz8MJbQHovl8iPiiX6pei\nyOWpYqYY3woUMYkmbwblAe3m6tgy6qPIvR2MuLn/aQjsG80QiOmmNZ1RHK3K\n2NJ8\r\n=2pIn\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"07f9e268278aa2bf4013732c71c48a5f8e64ab89","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.2-canary.2_1601333918449_0.2013208042650838","host":"s3://npm-registry-packages"}},"0.23.2-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.23.2-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.2-canary.3","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"0a3acda8b43cee078020b5b4beee9c9145940f2c","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.2-canary.3.tgz","fileCount":27,"integrity":"sha512-i6TGulY2P6azzsLCB7fvrj6fSeAL5Ou2MEu2xONsh/2bW5jDt2h7rWis6qvP4A6caHrCs5wxaclB/75g0Yao/w==","signatures":[{"sig":"MEUCIQCqT2TOppvzrHy6szHVREutWZhPiTGTdHzATMWUkCtfHAIgAJiq8pZ7CRHSPro9hBRABqj/RsipSA6n0hy+IZJzvJs=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186059,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfdKk2CRA9TVsSAnZWagAAbZAP/03RS+XiweK0hlhBADVy\nyCKgGu4M7RWcc/jQP5TEMaqUQ+N9TzSvb6psdxi9Bq7jNLfHyVywVthCjGoE\nqnbpb4X/wPlff/K3mfEnmk8RdJbR9RW/IWxwryjK+77DK06pgUe1uq5kwNYY\nl4B47j9EBUmS14yunsbf2p8tRcE8Y5VRyVhJ+pXv9/+1fP/08WirPg7J0DXk\nAjCMCAoBi2DDiwLMRB0O0iwZV4wrtDJHw7vffBTTFe98xRpwP21DFZ/04LVb\n/SZbLNPOAGq1mM0d50JVdqlx4371hVeb5p/vl/6D8XZwfXa3yQOT2dmafizh\niniTF5FomLvZ/vsA7oLRNui2vZy7VYnlurspXBn7ulqGvz036ktZLivYecp4\n0fXCCCeijtqkOVf3wuLcPZj0RqYkycwz4uyLW6nZKDndquarwdQb4TeQfmsw\nkm4eV2FsvMJ+5vaL0oqtQaqxMJg1ZowCPdwEK4FiYYddYZGBQ14p/XWXzhMW\ncPcnSdq5abIrTYsFDSNjRg0qZC0+SByUIVwKXdHumnRrY1pGvDapDdyPffVO\nsd1asIzrDOMOZ3fFMcKufUTLhsm/KySoyMj+twK9p+WI09wC2xx3CbzHUMWY\nvvRniM7XudHiGL7LRxEIEZEcm5ka+X0apAhUHEh8fj1V9jEPKiBmhbndhlBp\nBH0E\r\n=Lfgj\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"90df4e840922ea22e3d7e70e0ac90c4a0db64819","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.2-canary.3_1601481013727_0.3797053399273391","host":"s3://npm-registry-packages"}},"0.23.3":{"name":"@blitzjs/file-pipeline","version":"0.23.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.3","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"092bae34fa297e5366c5d0a01fca1e2bc8738095","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.3.tgz","fileCount":27,"integrity":"sha512-8LF8ynLTLX4ph0LUiguKgZOAoeQw+83GFGTJsvZpTwXfzXZPToJJSXtL5qiVzCTjRjeOykrFPRFuVMJBe1YwkA==","signatures":[{"sig":"MEQCIFV++Pp60A7f6i1eb34wzqOQsw/6Pi1hfcH3qkE+wf3YAiBZ50woY1mTkXxPx3t6mmIssZ8ZhWBd1wrPCOM631SDTg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168778,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfdiFbCRA9TVsSAnZWagAAEcAP/1Artt+NaTNlyo66bstf\nvuoCrR7I+70K5kz2Ax76DTjBXyBcczXNPBblA5TnNOjauM0JBeCIXY9X3kik\nKQeGOl3GsmUhTi+F/+j3ur6beaJtdhugApYqw65iweFjp2HqG0EFg4sZE2VD\nb7H6bSelE4ZjkcFr9eJ4eyHeYcMCp+jXTxGGlEAw+QkrnCR2qVnImrnGtu2t\nhEPO8j+iGpM5hgF83z9jrqZqb65N4kM/OWyyZUc6K03vLZmwNXC/sUKI/qRu\nIcmf8wJu6gpvox6SNCIUon4FXHzOZclC9SOH77qe457nL0121JUv/ei01kue\nfo+yaaqAEhGsbV7ZxrF04Qq+tvIsowbqVBRCZ8UfQkRWZVnFWFwgmHxmPN/+\nExmK2VA6udtolbKldj1pbAgBpB1I4v2FdweGMrOgx4fA2yQ8XdtbesfyL9sn\niUrw1RCpm5mHXCfL4vTM/eGa7TlhTrz9Fxc3DngC3nYbuPPWGVxEaGSWecQh\nns8cd82mEr3RBzXvAk52zR5JecBSMOMvpSHPo026L+785odDXffQEzPLXsUK\n5e8SUITIr4Gficpq6MBrKwmImlk+V8RfGi61pEPkuO7ZneDlGUihBI74io+d\nhLHHZnxVXXCKni8EvOWOmgjquRvWRX7VyRB5SSvC5gahYnjLXdDUcnKgb1cY\nXmyO\r\n=t4vq\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"ad2583c44e283a5f6d5c0e7d0e543714d3be08d1","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.3_1601577306917_0.14778764400207978","host":"s3://npm-registry-packages"}},"0.23.2-danger.38af6c59.3":{"name":"@blitzjs/file-pipeline","version":"0.23.2-danger.38af6c59.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.2-danger.38af6c59.3","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"35cbc87861f25769fce3ca3cfd54e5f0887ea58c","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.2-danger.38af6c59.3.tgz","fileCount":27,"integrity":"sha512-ICo8DodUUgMV9MTI3Lcq1RtjoQugK2btPna59aRamWAm/GXYt40w3O/t+wd4IdttV2McQoH4+y8p64qFC7abiQ==","signatures":[{"sig":"MEYCIQDBRVWWOrxVrIm3cEeYC6Al1Oeja3+xB6TS1tpNN2k0HwIhALQCjSwGVI0dR3Az/4JK+d5akLz0sGAMVt89GJ/r7VAY","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186077,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfdnydCRA9TVsSAnZWagAAe40QAJAnWdlMR7DlIh/LuPLp\nah66DaUmBDur9I4v0zu4OOR1eohYokG6mt/5Et6c9lOUThTZ1gLiWWzwUFKV\nN9j0EixRpW8KfSGYMNapHh7L9PQ76+NqhNK3mOfvq6w8Z0OF0Do5lOkmIzTT\nXdZRKW+xPIzzihHHWNg7gvkYeH4m/2LDALFNd1Z20qrOscu7Ai/lMCLriw3F\nFmcumYY1Zqfgw5xmNP53AOOhcnddxfMk4Ww7njTwq7Hmrr8y2ZDB3EnkgXK6\n0lFnQf5bILHEdMVQnFojudMWmw0S2wtjUd39bE2HGSsNb9e+vhho/BPWT2Tj\nTc/NiuCUOesmNv40t93sqna1GCrTg9pczNpo9jyWOoI2aJrPwRdJC2EiZZiT\nEDi+4X/qDiiuJyw5/D+cq5hbGJy3+9BX9zSl4Adl6G/Rdyowq7VD/FT4wgno\nZsj09tfWnnWkrklEM8PHN7SLSMWuk1zwsI0IAbYQWm9WJGYmDzzG5Z1FYEAu\ni5qNUJRso4ZbPvbP+lO0gtzZ34MsuZ0k+IdhuYi0YqDIkvC767YJiCNFOVNs\nbQiSpTEExYiwMROZnCdF69w38wqo4NYyTz2r+y1EKFAJB8GhiDDTw39HP2xZ\nfEMboNeY3TssYx/yRecMG1WZlWPbE6/JNCP78INrRQhAAWgp04OYc9gQufVX\niYQA\r\n=FLY3\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"38af6c59065e1b9d7cb51f01ebb48f54c762b3c0","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.2-danger.38af6c59.3_1601600669434_0.02646698938459613","host":"s3://npm-registry-packages"}},"0.23.2-danger.d81f84c9.4":{"name":"@blitzjs/file-pipeline","version":"0.23.2-danger.d81f84c9.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.2-danger.d81f84c9.4","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"29c04d676e2ea9dd1340bae1f56bbba285f9f224","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.2-danger.d81f84c9.4.tgz","fileCount":27,"integrity":"sha512-/IcuUdpebDpTkZGIrJBdZnVdyIwaXlENHZqDMDkKgXVq+8bXG2rKJ9w5Kfqmj8ldhTh5uvmtCVM07cuJtPysJg==","signatures":[{"sig":"MEUCIDSrFtjrNcoXWZxPq1IsRGftA5isYP6bzX/tHXRFXLuQAiEA8q4238hCZ5YpSFnYAvYBhGRDs8OeQhUZZVcW3o/MmEk=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186077,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfdoqfCRA9TVsSAnZWagAAjIEP/RUlu9FTBIgj4X5BWRqs\n/U6kEVsacIKq9UogRDH4faLQ7q5BtwGF2wEn27Z93vwXL7cFjlHBCCBlJxOG\nCadbfSEexSWZwG46rhMxot2jSxdTqsbA4Uk1cSP6WsH62HP6Q7eqBBW4zObf\n+/QRY2LE0i0ovAjKtJ0JP7eHg/pt3edQ0xyf5jDude1/RUemC448CkKqCFh6\nZfbbjdILCRY9AoHG4cSFgc7oz9pU7+vfSffYa7JRf8yzV9ffG06PbtlyTADe\nZgoqQl1jrEAspQ+P2ct8o9xpehz1LPKVFdIAPH18qE9IA2KBOei+fpwFFn3y\nvQ572O4FPuFIDquGHkcl7+ZVdTofiB2nprI6QYn0RwMurEY2kKDH8StwRQna\nVaOz2FBSI+SI32JUkFmwwJRz8Y1c17CaDh1zW+lYCpzdNRz+E6EghJ9tAPNB\nFe6FFLRvP0kP2gScW41E5cr55yONahCzFxFi5fk0AxiijTLIvwYvHvA8qZR4\ncsa/TC+S7BuCNtPwENspsKrXtAWaMG0ptzV7pjz8fIEy33xDugpCUxadHY+/\nKRGxtw2gFYNgHpzp96OQAojCoh4OgdBDkLIJWqFW8LymFD6Umjsfqor8eZOQ\nadTDwLtyRq5KZeKfTSsLXxpt9VceWMsq+SdUeRaohQxh4ZbnIKsMFR4alUR1\nHyq1\r\n=SeMc\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"d81f84c9c504da3f6c31b85c8154d4c529d26b9b","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.2-danger.d81f84c9.4_1601604255392_0.7576534410719773","host":"s3://npm-registry-packages"}},"0.24.0-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.24.0-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.24.0-canary.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"7aa2d86b6dbc8fe12a508aa96532e67e0eb8a9b6","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.24.0-canary.0.tgz","fileCount":27,"integrity":"sha512-ziYU0/NtrRl+hPFHV+hXJFQqImPNlrdC9g5sz9QUeOGvU9tifUPCo440u5I1deSeHWDHbwp0Uzeqv+T8sWI8sA==","signatures":[{"sig":"MEQCICRC0UM0jGFDdcP/SrYUXMfMDg7cmGUoZvcJNUF2S8oZAiAixcILxJjO7Fq+RCQheEeN9jQxFmrtNE0X94cZvWyT6w==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186059,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfd42sCRA9TVsSAnZWagAAh7UP/0dlaRX3U1nz+hVs0+4w\nrDyQvFRRTWW7I6K8GZW9ZOcp0C7ByfWXpO01WtINSGWwiEYcGnVvvAjud4v4\njHFoGbeLEckVB41/GTROdha9f0GzlMWz2qtOx85QS4FNv4Dysv8pcgZNy5xh\n7HEutsvSgJ3cuziBe0VUVovZHEdK4TEVoYy4+nRpZEvVCfHnO0tJ2jC83HYI\nnbUfwDdbP4d/M+zpgsIYPcbP0D7Nb1/y0NMQdZ8OccILMNUVXspvtzpXRFhf\nrHfvW1WyrdB9YiXCHKYcvQ8OKz+YOi6lusbEOs9j237kSEmDW8pq4hah2gSb\nLN4TKf9CuKSUsJwhSIeoK/rQXdvawxxisn8y+JDx1reorHgnKUC5tOZW1KG1\nmtdOnCPoqLIu4IZnV5BRnwa8vCPKL4Z6tBnwDedL97+V1qMNksd3xDG0Z6tK\nyU4mlhKx71/Hjl4CbMfkGcjtI3JWNGbKtAsMiOZdFbmQMFKbeHebV8JAZbwD\nD4oQwAA4RLyknfcpnwxXbRQ7yjLT3M6vjiKBrInAcVKjXAhd8jif8T6E3HMY\nW2wUW0knfnUALmdJUUdMntS0c+Dw7C5M2aV6BcszUK7be2nrYrt1LmypdrK6\nw2X+MC7t7cExrkeV3jRdz81MFWCr8SWrD1aU/NaH7hiYMr7PoWj40XkZLXlS\nsbFY\r\n=yGE1\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"a670693e9d79a50bbe0d48d4fc2a19be8553a184","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.24.0-canary.0_1601670571505_0.8708001425186496","host":"s3://npm-registry-packages"}},"0.23.4":{"name":"@blitzjs/file-pipeline","version":"0.23.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.23.4","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"ed5f6275cdb2e1d64139d6043efdc9466d243593","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.23.4.tgz","fileCount":27,"integrity":"sha512-FtApqZTYLWwGg6YHfGtfDeF5Qu8nO23dfwduZh8DGKHbk64/rbAIpk1GVD9Vd3niGr2FV6XIXHYJsNVk+iIJUQ==","signatures":[{"sig":"MEQCIHyaLW4sRl2dzxdQq0VQ9scGXPD4RG4MTeWV5jhayvX4AiA44ak7rF2iQ7oivDV1zxv3EiQqqFOEbQ+MV/wND5my4w==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":168778,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfd5PCCRA9TVsSAnZWagAAmdwP/jzvlgwk1A9UIV89IOD2\ntG5iE/iZQAsRVreGp5gXbrvz/bnNVERE0LIat5dommj0YvaYJY5RZE2NuhEa\ngeypmsmJTyoqngG9F9bpw7QPQ5Zq1SF6cpITzbC4NSibgD2t6HHBGq1M1yn1\ngiMQ2qfxiSRC/MkDdB+md+nOkcSZZZVPD+CYWOZRrE+53+J3fohVJfm+U1TH\nDE6HRlv0bQ2yanNuIV0IngzDH3HrVErUm5cZ2lyjSiUGbSN2Kf0SVvQLyJB3\nqngM8PWsJNqmiP4o+jEhaOOqAkhyrhPMJ/K4MqPG723gj9jQCjmlv/53Lrct\nFnD5pMZayUSOhbKGdJ+Ob9IfO+7HWI8X2JWXzM79joDIduwmxNYoLW/vWlYm\na0qNbxIHueylGYebmUsinjr10vcSrhPNzpQuDtRUCDiwjbdCXa0RRgA3FQoc\nRSYX95ypGA3+2LONZo9FL3mRxkReL88krefH970t9v1+yczT6t6grYICX4Mt\nd71g8eb3FjnpHPeI/W4SVMb3y9zg5gxVz7LJvu45EdId8/l9bd3mQDsg0pf/\ncrTNn/fhPJuwL2lJaPaGmulibnwr3bCZ8gclcVddSW9MbdgHM+FWtaNrAgFp\nNxwv1lwdqy5/EOID7Y0cB7jqust0szglT4srS6+6sshJJtPMRjGh+BEKTUvI\ngF1C\r\n=syjl\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"b9f0af1ca6870a5f396067f7c039970a0b2e2863","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.23.4_1601672129614_0.5124496199924646","host":"s3://npm-registry-packages"}},"0.24.0-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.24.0-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.24.0-canary.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6fdb373f2f7a0132af780b3cdc174b72fe6b6dce","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.24.0-canary.1.tgz","fileCount":27,"integrity":"sha512-+ErUZo+Pv1L8IG/NjZPiu1thdI2FXiU/9CyO3iKLcBV3odsSJlo545eNGRlnJLXIl/NQ5297T/XIxwCXV+8dbw==","signatures":[{"sig":"MEUCICU8gzzx0V86XjDuAZg0VkP3m+ik1RHS8XArd31fOqvuAiEAzAGmZUzGnA3gKxV2rRWfGas/k+eNhp5h5/17OERTZyU=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186059,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfd9quCRA9TVsSAnZWagAAKS4P/3vq9+0aJ+R1Lhk/W3sL\nsHs6u16FpldWQeMkhqafiCwqYUKlLR/6KwjvmfzO8L94xklCM32ldTBeULnZ\niabZGdXTAcdD2rVX6mIksnpqqm5wa2zMsCjMq5NmGnFdfekDECO3ruek7GYu\nB1ZaamDEDpOQaeFSjrv7ZGFcX7L+wObOwb4UT2caAhIgn2wXt0Q/R9NJ+8TF\n3XRXimjobuICDewlSgoRMCPvch02NxqO0hUzoaD9w9bh4PsJwVaWtynfW1Mj\nqSZftvNie+QcyQNtVxbd/0yESHUlnwZqKhNT/yURW/muNsHQPtMMMkswjcFF\nwtJbsw26/rEpEvtJx1t8ucQJkmY3FZlJu6bihf47AihxjaGZUw4a2hrELp/m\nYDGYvO0PepnMlA3FY5SPvNPRVpnrpAtA07ASTF69c+cmVxj9+JZ6XjFhVN/y\nvJlSwPsSToDMfO9RYUg++yDzEJYRiwNnEVNwNa4WlcxJpNQg18gxOdu79u5+\nDiLnqWZlDJMC9sW+dgXnn/swvYGbgzVrK5Q+RjqTq61OXuFggtP1p/J27qUG\nmPW/AesMg6sh6sr9FBLn96wyEf9+1iVqZsdAxt7cNrQDQJbVGOk64fRYWbk0\nwGmn4EkoVMsTMVQ1nsYDPbDAk6Be7b0kglMedyP4DnSJLklfJHsRyyc4ZMnJ\nyhar\r\n=kVpY\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"ae6b22f4f098bb96e1873247f3c1e5f447ab8556","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.24.0-canary.1_1601690286132_0.8649835419930614","host":"s3://npm-registry-packages"}},"0.24.0-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.24.0-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.24.0-canary.2","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"3fa4b93cd2f444bd006f6ebdb5a565a827ce2a2b","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.24.0-canary.2.tgz","fileCount":27,"integrity":"sha512-YthcUvsGNKA3HIPgr9sHlT3F3o7XkKnoBdo/768topgWJich5FWtnKGSkrii8RkjgC2bhzhu6L1iRU8AlxTcTA==","signatures":[{"sig":"MEUCIQDmfDNGLm6ViQ+DdHq9hMobv8COnFMORTbHx7x5Sa7zGQIgVrYTroVe5h1IrAlkz8O9rRsecqtrlhInBQ0YDGopLbE=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186059,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfe6iaCRA9TVsSAnZWagAAgRAP/RbYBjLAdoEMCMX/vbNC\nOjIyMtxkiwe4/1ceacxQxBShhak6JlX1llr44NZFVSJAmDvCuUXJwelsrRrD\nP/nMrenlFpwPPJB9GgxNUE6PlEQrBlpbOMNc+8WRUgnCtYBFYM3Vt8Phg3F+\nuqSQRy2s45OdDiAUtNGvroJ6UGQFu6jAltUrArMDPGWn18UWwrG+3Hf4OD7Y\nbPJMEev8aodHD7/YXiqZEfE6tUt7+gNY7HdkRVS7Q3Ci8zwAjgMYBMnh8prV\nGhr6BChyqKlwcL0b7R2pSZtdavaxSO0nQX5dgeYo0Jzbx2EGnnFqnx5oMxQ3\n4Cqhut5z7VhApUrQ2aTGkRc6xC6fSPpT16qUu9K/G9+I3tjUOgHo79cal6oE\n1GF1L6NHOsfg++v2HRt609m83T5XbUVQQxJRKzrF8BAWpyUcdFBfjOHA1f0s\nJcgikYd1HS4I8y2BcM/Cv5lmG5IBTI/Ko7/AzGSd/5B4O3eyTs1hiewjwFOy\nua/TRARhPq6cP5yMiTwX689KR582kmsSZtC4OU0IHNwmGRyrD8PPmOT3x+TD\niCZJx3dhydTqvPExsI9uZyqKZce5nZ1EYiLO6JYVQRF+DmDaWSgiPEvU0bAx\nJihI2r3FRBHn2cqQJtyLk5SuPlWbK6E0/ez5zepyXIqmemQ/0K4cmaY5Pywi\nt/ks\r\n=O+NU\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"be861c79193bdf41b059e8b7f915cfd8cfef3656","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.24.0-canary.2_1601939610266_0.29725573214578116","host":"s3://npm-registry-packages"}},"0.24.0-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.24.0-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.24.0-canary.3","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"074bc13e0b781157f141759ce66705e4fbe71012","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.24.0-canary.3.tgz","fileCount":27,"integrity":"sha512-+Ezon1I44t/p9h6+CF3uquTIUQ1eN38ppXQhRMAzjLxTYWxnB65MV0rFEFnttLe5d1g6IEABnHzzJoa3vObK9g==","signatures":[{"sig":"MEUCIQCcUWBZFJfSqY9cH+hyDnN33bsOKYmBdKBdSHB+sBiC2gIgO7nF31SY3MDgLfrNmxVs1nPedUD7A4yKq/2b2J7y3Tw=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186059,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJffIwOCRA9TVsSAnZWagAA7mEP/A2+RMrFpwTuYwMmMblv\njIUgTRtb7E45Cf7vv6FWjdGga9+Sxzp94QeQ2MvlBOkkdiW80KGF5Z+Y4lk9\nIfwFUo7n/6btgVyy1YmAONaZ+O4dW3UOHEA2s4eoczEfN9brZtw+UXsKdaq8\nQ02SoG2QBcgzZ+auGG1VNwJ3bxmaDV20f3IND483KXquNc8ubOrYozfzqBdc\nCi1hsLDVvV4iRmH4fKCZh98Nv8LtF00dldhcz+YT0VsyjKnp1jyApqQWEzVH\n8CmrVbFgE97UT+Eny3YlXdKtDjzFa9Dlq3abynK61T6Bre3lsPfV6/GxSeX9\nlxaJmx+HTH3d0trlA+pv6FUVHhXKXWxFIxL8izu6ScqZOfCZU1hPeNQCTLOA\n8sEvC8rImm/jSpAbF2kcIPNx9kNNpWDW3JhTx4zx7A/0axO6Y7WDyGyPnU3/\nS1glIxL6ug9BCvTfXqkdXEOpbLD0vCeDgB9sx5eWZ97fJPFi8IHKIQ6gWWNj\n4Qv3e+35/O5tWNT+Ah9IQ4nRprcj1b+NC0hPx8HfV1TX2xeFi2diEI188h3A\niWpvbxtjV7OJzHipjqeXRoWOiSsqRQPil0aBf1ht5x3M0sBK1HGXWtxKtmPE\n/yILtLICmTrAs3FbCjzQZwzll1tmeEEcmIqSCmsPdglrziE3YPFtUXT0OmL1\n3pHA\r\n=5vMT\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"33c7bec41f0e9abb96885b90deb7986908a58d3e","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.24.0-canary.3_1601997838059_0.9589200967753426","host":"s3://npm-registry-packages"}},"0.24.0-danger.e5ead166.1":{"name":"@blitzjs/file-pipeline","version":"0.24.0-danger.e5ead166.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.24.0-danger.e5ead166.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"2d55a0d2200ca83124453205df1589e903823d3f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.24.0-danger.e5ead166.1.tgz","fileCount":27,"integrity":"sha512-stTyLKMi3cVl2B145HzQ8SdTwa+eF4GeNryN8cZrVCyZSGaWe48jqBqltbm1X4EJclZlt9as1D/HocXmHU5Bpg==","signatures":[{"sig":"MEUCICgLDdfKH27M80+XeX/VjRvmwFG7Esu3RGg1Som5hHfNAiEA+OCn2zR4xkkP0Hzb2yEu4k1f+s+ZXEKBVMjtOT3zxP8=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186077,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJffJNCCRA9TVsSAnZWagAAqw0QAIPO6U3D6yLzqN2BJGab\n1LmmSwgFh+sbT+be7KaqmTBQSxMtWKBsrpDaBXl9MRlprjzwBvvCehkwBkmo\nuDT+5kPRGFn0Ypkv2Ic3rlQci5rGgeBa/XHd8wFMTu5n5LhTjfw2hrb6MLrI\n/aepGuRbZQjRdwJXQWrqlOwwCT6AolhQZE+rBZdeAgGX53hI5/A5lxMpUYsG\nUgqC6XF7LefnJle4XUTjEBMxNs7OUf9dmkTvbAPMAFtviaF6LVBAU0XUptQw\n+JO6peVWeiODH2j0rCA14I/zEmAN6kuwjnP1hf4Q+tdPx05pGAZxdlO+v+p9\nNK020g7RApvCdkGSdjjv9YiMQU6+/FQcVrvYDT1LLZeNo2zfx5GWB0M5bVHT\naHfp4/t00ucV4WJU1F5xHoUqc5swj3kQbw9aEbaPuMorTI0E1s2I+QNnyLj2\nAuF/ygnZGsA30QRoQIpQPSfDrMYaSEY8dZm8xf0CR8grHmL45d2hpPSzDoJ8\nDRQjEBraTfb+d4Jwnb+eYMvalQ/k7wAYXQS/2ZmVmaxUAA/iTJCJG5A8+SuH\n5mKIXIhWkg00xgshg+lf1Pwqv9znnxQ0Xk9W2tnmybsGRjWZIh2DjnzARmAa\nXEgw77u6n60+BHWtBDY1NM3RTfr4hcr6ENVYlLSuxatvz5I1AULPLXhLMKM1\nWLKk\r\n=RHOT\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"e5ead166aced33e741c01038eae74dc0499e7873","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.24.0-danger.e5ead166.1_1601999681757_0.3764013336344272","host":"s3://npm-registry-packages"}},"0.24.0-canary.4":{"name":"@blitzjs/file-pipeline","version":"0.24.0-canary.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.24.0-canary.4","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"b152b7e92cddd1324eff986b59f55f6308f87c9a","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.24.0-canary.4.tgz","fileCount":27,"integrity":"sha512-hkqSYVKYOZ6XX91ZJj7NO3Qjrp+ANObt5i9sPG4BJp0X3PJ5uKgO7gZN5KRHk02noi5JBiVGin7gwdT+EaIF4Q==","signatures":[{"sig":"MEYCIQDe5tZiUBJkFxhZiBk1mJTMzt3kkFOl8Cp3T3BKGPF/pAIhALD1WO9eQVJxbQ7ELWF0SoZOBsSmNlXe2i+AO4bHjI8B","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186059,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJffJc8CRA9TVsSAnZWagAAyi4P/2eihSBsHZn8HQZYHxgv\n0xcu0kedAwT7JyApj4lNVWt92JruxZKtpuyrFvTPLfqfxWrQNRn4LMGbZ0qF\ndWMGr+xPw73zVAnPj5IMVjJWvr3k+jUuTA0k3HkhrBQVipndeMmPRXIqDQt3\nsYzN8UoOLJMls4CvUcWEAUO+p7mCAy/cIu9Rws3+/UWkhgIB81kRR+aCVGgD\ns29rqCUJZG7vgyqKKVW3CQF5Wg40tfODhPq3/hBXWo3FwtgrxoiwyMLKbK5I\nGf0gBChllKPlsJdnVTlcOC7AI9kofA+gcqhTLJaoV4RdZtab5hqpICFoCLbM\nWs72fxNue8hjWz0jFgfSUMNqQwSKTcfbrca2CF0Dgy/wKY5wEe5BhNl0NDsa\ndGaI1mqhd/DvMsdIcYHmwGy5psgAl7LyooOcd9dPpEXtnTjlIppl82n9BO9j\n5wgVVZAB5uY+A2TObDz1ITCEno9t01qx9nM6z3OtTgnAuTf/GeU4LTdGaVkg\n9UDpWOJ9dn82Us2y5OuxvFkqsZQJygJzyyKUkb0mjRllOfaUuQ4s7WzMRcCM\nAaydv1Sawlu8lLDV3tGZV1/mf2sVEpnqcwF+pThiYTcz77bc+/B80DGVtTjl\nIlv6KRB+UuBRNIE31IK3xBKDDTdIRn2Y7d7nWiET4M4GASsOqMLcnUiB2QRD\nwkw3\r\n=LJ9s\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"968f1d0cb913db34b02509690751cd4725715565","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.24.0-canary.4_1602000700197_0.24126192573130933","host":"s3://npm-registry-packages"}},"0.24.0-danger.e78a8210.1":{"name":"@blitzjs/file-pipeline","version":"0.24.0-danger.e78a8210.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.24.0-danger.e78a8210.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"7029d4cadd2181f7edff930dc6856d619c67c91d","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.24.0-danger.e78a8210.1.tgz","fileCount":27,"integrity":"sha512-R8fGsMX87I5GKFPx0+mLsG/Tlfwzr8XFPlg/zbc+zV24Bj4zI1ekKlkex0SagUp/g2AZTDqs2C53OQUeCPvM7A==","signatures":[{"sig":"MEUCIAeFnJcKQZ4JdUTmU44hnRPhMLILTLOrlaoHea/w6z2kAiEA/i+MQgDXxO/irUCwXWFLH01WfnI7gxTs8H45SsXCL9g=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186077,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJffPFrCRA9TVsSAnZWagAARfUP/juu0TFQo4VnzL3GG7UA\nMuRlpwpL6oMk6kPsFo4OHW6u4c+yHEz6Ems1q/DKsXoMSJYXubbZn1IQMSYe\nELCZPc8MWkOkiHz8jM7pJ1CO3MZpe0urro2WfDPhpTOX9iDk5+1qkmIfdLoG\n8unsL6FEqJhqsep171FR3pfLbqglLFFd4okrFbb+iLW4mhhHyjYRKxEnm12H\nZB1OzKSM6NKuJ4i6W6qOzfR8eWEhHGm4DQsBT+DYRLZWU0McuZxe/F9b8Df5\nmMGcXqUUowgDmkQqjDXZZ5YEdEkfnLosBkz83enjF7Ww8SILwJErneqruASP\nkbJa9esHafqbBd+btX0PJsDyU6xk4hEPtrzvkv6HW7/paKNgEkNk9G1TmygG\nYpZAgSGTcBlYyBMqg3tapGwDEo3lmB80YRabdQ52OA4dRIw9xU4i3OE0hldR\ncHLwd0k6dV4ceu+ZSp6pGBnGNnvbeQzbQ8+ueHXsuk8+RshHVOUmm69dvgS0\na6BLOHzIOVf3dPNs1DDhQDe5R9UJ4nIPOYCJT+QgXXxw2hhY0+a/PZgt7grL\nQ8102PV4EaUlpe31ZBr1UAoNm2hY0ZR1iOcgInMmaxnsk3fyHTvkYW46Um1V\n0COldcV/pmCB5bKM4Krl5T+J6qL05Z8T1kOYjnzXMnkV7WsPNle59Kv85MzS\nyhmQ\r\n=lGUS\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"e78a8210d9041ef042ec1d4f9325bd5730307019","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.24.0-danger.e78a8210.1_1602023787303_0.5116127496975571","host":"s3://npm-registry-packages"}},"0.24.0":{"name":"@blitzjs/file-pipeline","version":"0.24.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.24.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6deac85e0172ffb8c818cb55b95aef266da5019c","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.24.0.tgz","fileCount":27,"integrity":"sha512-1wbw8jhniN79egevzRu0ZYU4mHkzG7frCT6Qa5fBVlL4/1BBfo1KGI4F7/0gvpSDVUO3Wv2eiosHrPEhpk0vag==","signatures":[{"sig":"MEQCICq0kVTnoFW2t4tfYh2+hWLx3AQM6x6jkgJyg8nmHBX0AiBLtIGFusldKtFvXb6ko7tsHwZnLyDNggyRjR2thxU3Lg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186050,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJffPzLCRA9TVsSAnZWagAAdD4QAJ8xWvrswuiK/wBm8wWH\nqSCnbJWuoAP4D9qIhil3c2ZCEB1c6v0l325cIZSrGUy7C4Phm1qnmGSxcaC9\nkw1/nsrroKwc6+SnNhd+NAJ4Sr1zSVKh1EZoyYwt7FprO7kA51fGxVcybE1A\n4imdLnXgGJuq5O2AULik/w/wlY1epVgzU2XhmXR+fFBcBGl10Z4krXjkJ0/X\n4+an8aITFM3xL50DpCnwb52lCE03VkrmYsLGoRVuFaOFYUCNwTK21LE6mOci\nJKxb8OSktB1jkO+OCesWWZnNzaFKbjq2SGGMgafXurYbqJdsK0cup9a/vsZO\nUFH2msCnmrTyQnRGGag6gol0DiGF1Z/G59rZhRC71U9IuQr07sbiljFet1rR\nquTFCPeyCk76BZSeC2XT/sJOX7kavJXDqtg98vLllL/rj7/lb0vaTa8CUXKi\nAmH8M3UXLS+P9Zb0WoGw+Z8sFtRRQKh+7fLS84C1xNqo85myGQxtHn31VlEK\nvTE7kChv6mF5DwJJ/CrOPOU1E1Py2kMw5MVneaaa/1o19BlgghYON2/xppBy\nHdJMPrEtpDBxzEKnl5068wBx14yAmxFTJDXbX+yL3IyMbMikmiAqTjqcA6ML\nITiaFsUZU2rezhwTHIQ89e0DaimOws2NA6pUPUvmP/L2q99PbRCc2exrovZv\naaov\r\n=bivB\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"99bf898cdc7f37e431b0c3903da6dc5fb66d8aa4","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.24.0_1602026699320_0.1466273537374081","host":"s3://npm-registry-packages"}},"0.24.1":{"name":"@blitzjs/file-pipeline","version":"0.24.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.24.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"67187f0d6bf5568320c3a59b833c1ef962b2575a","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.24.1.tgz","fileCount":27,"integrity":"sha512-R5PtpyGYFnc9/fCzLyLf3aqsxVQ2Rztd2BZ+Yu3fMbqWbIPgTceP9sK3scpXg5Aa/kmFduc+c9NGwyg8efiy+g==","signatures":[{"sig":"MEYCIQC0vnnDRtD5cnT3WOFXGUUKlV7J0qE9XI3ZIE2L99ZqAQIhAMW/PekqzgPs3HN/uij/7CQf2hzP8LjboOLXLTt+9+IA","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186050,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJffQzfCRA9TVsSAnZWagAA2OcP/jYsIUXuVhx3yaPBXH+O\nfGrtEx/MVfaHiCwzEFbeSPoKpD6gH8sR5hj9VFPkbz8Py6PntuEGjWCsSkGB\nTel1xbE+BwUVS+0cTRVBU0jhSK7wrz33K2/Itzez1yRvaTwR3mt7dFhGMQeg\nUcPJ0lzOFTAgoe48VdVN6MleS1uM89kfX8XYlH2C4/MrfiVN6TYok5ol0cia\nq4S2s+eCZr9reHRxXqUfc/4LCJvXePIdYVl+5f8g3q+wQSzdszrctyQ4xmSk\neIU6wxLS53/PDR/8VoUmuCgpaHIbE+u6WcYnU/INkgWfYY1WzLM4yDV9qR02\nbpzLcjjtRpLfzubI1vG28c7W2YnJGcihl0bJGoarKORUBgFqNx3A86IZ5++h\nWZjXXmxmrKWBOQDVQ8+qzsO/zYkY6lGhYUW1LWVm1rl8KoqML6bkb8LrEKrg\nyRKQMUDAlSrtOdjNE2cFsWItXNrEvKGsQTLMkMKpyWK6lPRcZF45rnOwxRrU\n5DhaNBeljdDCCvFDwQRcX04saAlxunYTRK3oQ2Iq+9QJrUji8fZuKIUVQsK3\nmaJ63mp8K7/pUlHNI//cLzitiZB6otVxLtZQnPxGo14ZmdbIFm3+kyAFi6JQ\nLnXHLyYryc0uLr2yysFfKHZVEw6gTchqLyVDCjWmpfeO+zmpsvOJ5q8u00N1\nxAGr\r\n=NDkx\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"814ed2d59af19abee83a2edd9c09cfeb35c277e7","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.24.1_1602030815107_0.25262843180008","host":"s3://npm-registry-packages"}},"0.24.2":{"name":"@blitzjs/file-pipeline","version":"0.24.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.24.2","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"60849efa04c4406afa265b1bafb17491d9093bd0","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.24.2.tgz","fileCount":27,"integrity":"sha512-HSEquhGgZun6WkcoVaSliTFhJWqCFX1NoGXNrSSpvUGZehur3ImkBorQpwtrvwEwK2AWs37+eNaYWBmLqwgrZA==","signatures":[{"sig":"MEQCICqZrEfl+RvZN/xlHNeAQcrCwABqAyGN2Gb7S/YwUfQ6AiAlj7Q3/bVNJU395FViRKYtUfsA96Yeo6U7VBfWV7VM9w==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186698,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfhlAgCRA9TVsSAnZWagAAfRsQAI+txQd5NfKhTuQ162Gc\nM9APy7Js7e0VJZniLWcC23M80TROE4sFuUNYa04ZqDEKA5fpDD5A55zffULI\nVxkh/jbzkTvJhxGvc+OLba/Q0hDhO7+wQtywbnDDoOJQA8NI/wuNAAxfcmtP\n2Y7kC6fUCedrMc91m8AXXmR80ehf0IzIJd7bzS38ob+QI2JB0wg5zuimNR0E\ntfhF+9Dvr3wzK2r/ak0T8cr3mN/ktr2nVTXfHEeZBoSCutOae6jgwYTSVsiZ\nktO4HkvrPLWHE/Cmo/g8EjiK5EOgaVwB38HUbFSbTmPrjQX1nEp4TtDmLD5H\nMOUzst2Mar1K64HCurVl4DEcV8VExUPypZYKOYh3sjdHx4IQPLAaIpaitMhp\nsSeSuEt3OzLdzWEpdJjDpH4VV3GeOoFpXR9SGweX73r8m2KJGW2H3z9H/X6w\nf369LifFKYC5a57XhmxNsYkQgHX8U0f43czBVL+MBUR6IqO03b0huJH3p8Fb\nwobyFhPiLdh52cyI7s/7aLhDPNaaV9an37M/jIHd1poJganhg1IVz2FgBJvH\nXIT62AeGqad7Cb54Y5v9tumYSpcTmvrd1gP4z73QtCjvFXwen+dJU+9erBVY\n//tS75uXx6t1MpgQXyOUdXQ3x2xsP8lC87jJJOIFtiWLQHUgSd9t/4BxiahV\nc1b5\r\n=9PDl\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"66111f74bf18f25c5e4e40161fbd0ab8a4dfba24","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.24.2_1602637855608_0.4773998857251649","host":"s3://npm-registry-packages"}},"0.24.3":{"name":"@blitzjs/file-pipeline","version":"0.24.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.24.3","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"2ecc5de7ea6e049d7d1efdf30651eca945ed46bc","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.24.3.tgz","fileCount":27,"integrity":"sha512-dUhtOr2m8AUY4LnGpQ/pcD4JKMU02AIQ6HupK6zSSml8/lfkMwVS+WKMTMh6DcXcUgKXgIDKTGiUBN/2fU/UMw==","signatures":[{"sig":"MEUCIF7qDUD/oWaTiMPuCnfGZBfT8ctavRIulb5DFF9F8rHoAiEAoxbnrIxJlIa8m3OUMYHVOtJbDpolKZlqt78z9Hg0V9E=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186698,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJficVlCRA9TVsSAnZWagAAKOsQAILokwayXQAtZirE4Xkk\nV+rBZQs31bxbno/FePmimoNLQRpXZo9c0SRR4znjtszKif2Jg+mdoWzriSfH\nmXr5eDdcIKspQP/cUq3rx3pYNVw6qso+je5TYYubG7qooq4fID109TIBQQuw\n3EEbGCD6ODchRpVFsgfT6GLCOBSviGeq/eziJtxoii5iQbZXkW7puzYoZssq\ndHlaRUFY5oXYRTSqy8As3pwPiWArvou+hlCZvTGUw4xGIZ0CzRlLHxEmnftJ\nNJ7rnVSb2G+GYB9UW/b/HuVLG7hwjWLA5EgeCPMPe13NAKc8fq3m5jJiRt90\nloYJI9EDZcRsMqSm23qaxEIstBOG9y9IU6u0/33ev6i+Lf9ZwfSV0elk3RR5\nELDBMCBIJopDZjEvXcVZp4P3bYYBRXTqsH4HV0xYo2+9p2eTETElxF4rYhjS\nEn7JmtwHl63CPrE2dIKKBzKG+ee88sYs3ygNCaCB8cCMLmXFz6h/jG7xgUqt\nyPQ1Q6E7r4rO1o14Xs9XJuTqdTlAGGKmGNUeJ/C8+qZnfHCCeZL1u/WlwT6C\nHadZk1xO5UG58BPShfiTDyqnJ8frDcUX3hlayjQfC1uhQM0OzSxRnZkPWLnX\nTHglZKlXWqzGr/fK1CjHGY9TTPmDoNRATBZlfGFaSj1q7q4QEyPRbvbU1EMU\nMb+M\r\n=0RqB\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"fc40c9dc233da2a4356bfe6532245b1975f416c7","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.24.3_1602864484952_0.8354236774206036","host":"s3://npm-registry-packages"}},"0.25.0-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.25.0-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.0-canary.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"4eab25023ecc1a47908319bf793ac899acd29e88","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.0-canary.0.tgz","fileCount":27,"integrity":"sha512-jmUKokU3DS50eq148Zh7lIdba7UD3R68V7oWh6FpvsiOv4I1yj7rRTl4bjcbLjpW7cIy6dcBc0Dz8HBPUEJIBg==","signatures":[{"sig":"MEUCIQCrv94m451fvHEZEizCCsSixAnB/vETmRZ+JJkbD+V94gIgRFP0A3H5f1wVh2sw5GD2BFGzurdhYusLuZwUbF0ii50=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186707,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfkEGXCRA9TVsSAnZWagAAHo4P/jqkfAv/vzHeKYBuM6wn\nYwwAzstpj5ECJ+O9vuIBKPDamOunrCe5/CoXXdjyiYAclOeaGkqC3FBJmqVj\njszMV60ofnKIO4lB5b5sbfaJackIt7nR2py5AS6eVsBnPyJX0qlSRnBfGGfs\nSx5JkQhdejrFtMfNrYXvK7InWur4LfJB7HBA4QLrefGSZV10EjbK2MpAWssZ\nDquivftmJCyFKkhTSRO4tbCZ767nTVBzP8ihHdPpNrTYdp8/8UULHLcgv8Pl\nXcAHX+cxm+2/X9R3o+CKQXt/xb1tkZidJFKirbyBafYsy+WTP2+z76g6I6pp\n+Q1SWgj0noyW9ZkidZmyPtoyIvew9JBIGkVvqDmV6VsGCFEW6lAgj9mDgiGS\nbzELHK93T4HbZOVUcYUHfryOpFj9Ai22ftKgyQrSdru3dUTlhjO16hpd0OMA\nKjsmLef2L3BfOm8E5hvSesV12HiHRWelrtQBRXyj6gn6ESIoXfgN3oj0C2De\niPia2eO5+9+SH9ZDhet/GAYHTpalt2CvAR2ej3BCy1MuAYIwgswL8U53f0J5\nBTDjvpz5DVMaauonSInDMuWI0YZ68XPIPADQ6ar+wVlCeJnRFsVpFqi1lKv5\nDiCOp9Dyy7cHeZChH00JXt/gk2xLgEQh8QoNwE8tuGksPDK69y272H3+2juC\nyTqG\r\n=2LNW\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"f4eaebb52a57a38678e2c65def8945472b05e43e","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.0-canary.0_1603289494962_0.7228392635835521","host":"s3://npm-registry-packages"}},"0.25.0-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.25.0-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.0-canary.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6dfa0035f7a03ccd36ff9c2ebef93871bbd3cb1c","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.0-canary.1.tgz","fileCount":27,"integrity":"sha512-Ky9KRLM+O2/1itXnl0ZF6nJFU9cIT9I0S+UjdAtfzLuZYIzFIuZtzwjcX39CAmI4/jQ5XNsVwz1BF8z/vwmpsw==","signatures":[{"sig":"MEYCIQDAKzY63GLGfQY03VE+lGHKHIZvYIJvPGrMFuRRIC4hFQIhAMoDQw4Mqkn2CsJsKAkSksomp0RMZXfgRt7RCDS7p0Dh","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186569,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfmNTuCRA9TVsSAnZWagAARAAP/iqD7sKbPWN3bug0VKSd\noN0Ql6ZlYtPJMOjGXRu0tg93qa94krp4r0+H/iA2E+rzFAZAh1bvS3MsNDYN\neGTSsgXrfKpai833fiL/nuZzav7lxCSWFO3RO1R3VvoFTIXZxFTnnynlM3U1\nNVF7BgUerXtHtAfRdiBc267NfgpUwQoqQ/ED++7g0Tg+G0CuEI2iY3jEcHer\n75vFP90JO+VxosOnMOLa7dYI09fB4h0HasGoBMH94icWCVTiuWEKRe33FxIO\nJMNXwYdziNrRJzhS0eksnsCqAHLs6uzTps4/WALnmCaSChAC+CFUC7EnMBwk\n+Z7IFambzoK50fP0TPZ8u3nIgO2LKBRLdbUK2gle94KZl3loVMvDGjUGiUfh\nj0SIdqiv5l2MQqr2v/81GFjcUqfBuQKJKmyG51PQS3m1XysItjCvLhLUCcbq\nO/tV5zrPjdcq+XFxB5QIBYgoRJy4fNDfxOreedNaPGmRpSFhD8DCp0SQHhPh\nt3vwZpUkTQp0TC2ZiEbME4y/g8uF+lE85t1RN2z5kXsqJv/vx9ZfLZx0Lta3\n69IRLWdY8mpTnXcXiuWhCqFPpPMghFh+TVZom9LBDj2fTZfFwGeNrDQO6kgS\nSHFtourjiYhZFjFOd9H2XGG0HqT+jZVE/7bF01B2PQocoBph+FO/uxH/M9MQ\nWaH8\r\n=20UW\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"8f7b064cee83c9430bf158af6aa479b812b8e20b","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.0-canary.1_1603851502533_0.8489139127394056","host":"s3://npm-registry-packages"}},"0.25.0-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.25.0-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.0-canary.2","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"dcfe8095111e59fb62a83a220a092d9ed5a7a9cb","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.0-canary.2.tgz","fileCount":27,"integrity":"sha512-gHxKCT0U62gJ3MjAcCJVuEjf48ZsnG1XPimb/G6Tj714y82rg3+ad//5lWpLu/cYR3yc/zniS+he8O3SLdWenQ==","signatures":[{"sig":"MEYCIQC4hQmu0sdKmvWhF3AIqQgUu8sZYV/yKNZ9k5UiA53UbQIhAKt3T9uQJVFgHhKGriqE3ldpWeaatE1rXgD9ObAJ9hv9","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186569,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfo2pMCRA9TVsSAnZWagAAFAwP/A9lhnVWEYs57AGCJPph\nFxC293ddS0hcu6pVBiYT4qHcQWiCkuKH7i6mYl8TUtOaWfxeTOeBCuqQ0yUy\nQ+XZrN7K2iRY4wOm/n+neaNhxifPRZWvPJnH3Phmaa8HOLAPt+ivjhw5+5VC\no2cl10klN2LkJGXCNy1/o7VCjK1EdLCxWPDbHpB2Jqb1Nwls0RASGdCws9Rr\nkLdswPBjVwoaACeurV0uW7KDHlrG7sYMLM2PJKsAe56IvQKf2p/Npw2aDfHb\no4FZZqxl05vLBpY2VVL0/hb0agbqf3MewtArW+Ii60cmAc6ZARuLEvrg5j1/\ngoPq24caJDYTPc9Udhp7tmzc1nv1+0Sp/J4050Lf4VkTED+4cNbh2HMg0/5F\nioIbTkvcn1x0Gk5sM3d9e7fJYNWcXjJJNLrtOCTWFFY76/q5/P6xN29Dt9vy\nCXsfAt+1xEotH2ENZTYRABhj5KzcHwXJSBio214ziYxfCbpacDJds5tyVkiq\nc9k2YCyomtTL5PjA1rIHNFF5vaZwXnQHA3RtpRy4NYFGCZZrsJGb12DK2mwD\ngDyMoILlKXeoe58B4OO3mM+8Wi3x1z7uAJVXaD8eBwLUEREqQcoVXljwfo5a\nHPhgs6jOjI7tWTmh3cf6iJ3MO9hXeR/afLGjGH/EVnQCwkLxWDLpxhJtg/Y/\nbxPL\r\n=/xEJ\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"28ded0c1b1cba60e2ea89b1fecdf89a847410f9b","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.0-canary.2_1604545099991_0.5457629168151761","host":"s3://npm-registry-packages"}},"0.25.0-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.25.0-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.0-canary.3","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"e346c52216aa86e38ae16bcd027e88b9eb6bfda2","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.0-canary.3.tgz","fileCount":27,"integrity":"sha512-uEjJ5+lTRUnZlRV4Enr8Tohky0jPfp+YuQwS9SZjfY+O3bEY9DWzJL692Ami3/uoGH6fS6HaFi7c2wCR6LcLzw==","signatures":[{"sig":"MEYCIQD9t3WW6KopUGwIDU+P+YXx3ozDU8UkQCFCpEes8OXIhQIhAIyZqoyf6ZbfhN6Qb2BjbIRU2NsJaHpTUPyyEUx4Y/Ou","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186569,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfqfZSCRA9TVsSAnZWagAAuOsP/REFH1QnwbjSn2AlJuI6\nD5A/LA3pyGLbJRtF8WnSp0vhr6Fpr2T5nIiTUDG+i++JxxRgeWRrX+VVjFw7\njIYfITcWWjBMrsaSijXS0y8lvQRvrjd3HKHlbCra5at2SAopaq9QiguZa32L\nfaGcb/UBKsQChn5Fds3Jij4oiGACiKSTcJEyOPLEdszCxJ2hfzsuclChIwXo\n11r/3uIOQQRivFa7fm6Kk+GOH3HaPlxAL8k4li0nrgaouVxqDVx8xXKCWOmz\nNM8PoZ9Tk/PlthJlunURlw0t3liYAXNAVZm6+prrZRVoXqRqEQXvm0rTINKq\nE8MdkM4nl2LuXfeiCLj9Rrd72UG1ywDmKot38TflxTteYY3LdJtLFIj9pJ/L\nJSvbbnduA7wdcU0o0Q8AWvD/vqeQZZ2qpbauti1+TrNQYAwDcUepl9haOHQC\nVH/n7MbR1ViPvhEerDZ1jq2aJi1fCzsFHHYdqkRgepw3obhYi9rOs5RDsF1t\nn8y1qic0ZH9iT7Ue8MJN+qXyk9fpzGVowPjMy/1BXTEb+bbbsYUP2bGdWC4z\nEw9rSa+iovB2+7g9UqlUFTcmljxwKHTc7wGFdXCym4ZATw/Wnb9N4Rqol9GW\nxiAoeTe4BwGI2iHqe3ScIfZrQetE+/P4mxtd4UzX4EHeo0mWpIC5obYbrVRD\nLRZW\r\n=7HsZ\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"7eb7943b55e43f3dffb136abfade12948a1432b7","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.0-canary.3_1604974162133_0.289243202803698","host":"s3://npm-registry-packages"}},"0.25.0-canary.4":{"name":"@blitzjs/file-pipeline","version":"0.25.0-canary.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.0-canary.4","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"88308182e4f7c0109ccf62860b95d1f14af0bdf1","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.0-canary.4.tgz","fileCount":27,"integrity":"sha512-dD6+jCZdwir0YcwU6OXc4a8zUmO2T4aBbGjaIA+r7AkYrxoi2OyCRx5ddjYUHpvZ0x5EjyMIoRL2Qb3zVfvGew==","signatures":[{"sig":"MEUCIFVWIgju7jl7sbI5H/yG+H01r4RDs5x0pKURncy/fGX7AiEAzQ5ZG0nCnkygSHDB28V41UkpPyIy6FE2RHHjMUNXHFI=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186569,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfqfrsCRA9TVsSAnZWagAARZIP/Rqqo/P+VYU4hZZ23wFT\n/Jy2XW0Nj85i7RdYj5pG87VPDp3d704eh9AR3n8Gi+dc43Rc4cqLV0Q1Yjwc\nLRDv8I+dOMavzWgX005CQY6LLTtwFU9ymYsSwgGvEZbC1MqSrPEHpRFjhmA7\nwTe14O0XmZ9RI2skwkJIk8m8g4rKa+c5YRb1K6C6jLaocrhOCn1d4Sc1A5eA\nUYT/C65jb7D5xkQ2N7slJXJxyanJxqE4oeIj7rgx4EYjzRuKJ6jW+i6S299b\nYpCFJi9WMX///iX4UXQ7F59oVoewNNOoZkw2hSRIGQrdYiUs1vJGv0bhrjKE\nhcqPWYX4dMQ2QMor58qEbHVR0hWF+Mf4W06+I7u5iR6/ZZTOikKcjwSOzS1w\nwNvm6AhxtD4bNomtVlHC101IlI56k2MerApXBRcKI1py5YN0hQ/YIJPxM+H7\nHforTudtYl7QW+xhZUgG0ZxmrUrg7Cn/U0IeuSD4ZmlTeXQbavPmmnOQstWp\nQJeiUqfHCicQ/3Gp0Zlw0rJP7O0zpDR6An03dWOSlPzPO3V3Rl1Oxqsw4Ysz\nY3YQkap8kLMEK4AHVmR7Eq7V6HmU0Ey+MAJPHg4fjdLHLOt++odgyk6vIghz\nZs1gflZqmdnFrcDqpz4FEZL9CJ+KabJVTja8ZVBQIYZMkumIpzh7UgNX6z3D\n8y6a\r\n=syzj\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"2a4a209de9ea07822e348ee6e095835e0ba9ca7e","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.0-canary.4_1604975340300_0.4932942092120003","host":"s3://npm-registry-packages"}},"0.25.0-canary.5":{"name":"@blitzjs/file-pipeline","version":"0.25.0-canary.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.0-canary.5","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"97c18b635210f09a317e70c081d4226f1bd8b500","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.0-canary.5.tgz","fileCount":27,"integrity":"sha512-OIfdo7Ch1u3J8zWnD5Nnp8bX4Y4/r76cWCEXWkxj/ULzW8fsnjNjbYwc+9Q0PovedwDz3BvTM5n/vINgBnwWjw==","signatures":[{"sig":"MEYCIQCDuTCJoZhGLM+W/7mlS9g4zRG5kICFe31aYw+30A/cugIhAIVLQPzKuQsWO04uTghRgxcwQa2kCajVZLWmmS/q0x2t","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186569,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfqfzxCRA9TVsSAnZWagAAo0YP/3TBkXCvRweXZztJ1vYC\nHCi7Uh2Igf4y2ir/dxUmG3Zctrl8bn7DJO8Mu44gHQKcg2J895YVACP2VGq0\neofDV99R6qNpmCroS1Go1KuiEpZ1X6ptM+QCUfmuW5dJlpBO7w8sz1Hta7QT\n5EEh5B2z2u8maVXUHavET/7CmRzt/HkPnk4OcjgQrQxJMkpcEBnd51Em/uIH\n40rMXMC8NdB6f7GFp5fdhjOnp+rxh7oIPy5l4qf34mlL+yA2NnD4uR9Kzs9W\ng0BOKbsrT5jYxjsSQCUmNzzJWS/hrlGIg4Vwwm6Eev/sqZrr3mkufAPcOX48\n1vCKxVzbdvUdn2bByd/pVro/UFS4SszcZIQsxlJKQu9OnqWgLnbq5zGDj1DO\nHyb5IcHrWE6G7u9sfA7VDMIa8EedEs63J6AI4ULi3lowzi0gcD6WQeWwErkP\nfSMmrPGjbVErtLDCuNOWOiKrpp/vfwiMxbtgfGv9V7bn0s/s5TyT2nymDIX0\nU6zpJeocwn7TuxgE9nT+6o+GB9xAFB6kOgThMIIcGRrd8isddwWFBs0w5FDE\nBc0AqjuZjfU7tTt6GYQQAuBfEekzsuYyaGVxQSGMsizkH6xu9XcMyV8paLBX\nRIoqX8PaR2D/zWxwJ2wW5+aEjxH/v5WyFLixZgr5Ghe9CjLErvHMjAC9EuLB\n+e8B\r\n=6akD\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"680c44180c40e4b58fc4680f37169863b41eb9a8","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.0-canary.5_1604975857407_0.8676984143399349","host":"s3://npm-registry-packages"}},"0.25.0-canary.6":{"name":"@blitzjs/file-pipeline","version":"0.25.0-canary.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.0-canary.6","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"7eba06998b724d80d139069573bdebc3316a4ee6","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.0-canary.6.tgz","fileCount":27,"integrity":"sha512-w6xd7JqH+O1bVxy0xzwYVsixw58+DyYMIjyQN6pxKV2TGdWmUYnLv/K3RZ4HQm2KUDeGOmwoBV8oiqhflqv5Nw==","signatures":[{"sig":"MEYCIQC231dQuVynaNDksbaeRq+nKROF6PAwJy+IakGZjc/DBQIhANcMSCwClQCSXAlB9mFdAPSZsPhcZkbkITvcvayBh62X","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186569,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfq0TVCRA9TVsSAnZWagAA/jkP/jFxQtiwH4VgtYtvT/nR\n6i9of9EtaowVHQJeWr8FmgdCE8py8RNFajj+NClF+YXRW6JVB3sczhVyXW/f\nT5Eyu0ds5dzFKacE/5ddDcJrcHicctaUZX8OR4OinkrrCSSm1b9ACVwZ5SRC\ngxgvySniNQJ2UUi/9S4/Qo1SR3k/IY22KRjAubZd4J/4TscF2+8QnzvGHlcj\nA7KRu5KasnxOO6c2yTO0nDf/tJZlKmtmrzqaxrl4sx8rZHpWuOuzI+nwbYXQ\nNjSu7ItpSCtohRkVRNcowtJiuiEPElLfzsVmbVvxowuCR1U0tQ6n37xbRJUB\ncpdzvwA+t5O3lJRlIIhKGQ1gq33Bht5fFMmA039dRlv6p953+E2guFErs/Ok\nSDQcHzqsYnGQFqi24i/Qvi9wFgnDLDXHps6ZFsevCuZe7N/MFO59Zt6XbVD4\n1jIuG+wmZFhFnqsl1dZyY3VY+hqM2PnhRg0nYeERJ+qJYOz575GIZmO5Z2TZ\nDH3dSMQL5RoWGgPf2+IgAFtE33uLb7aGLG21vUMEnVjA4/e9zTsDnBgOImIz\nb+hFKFGdiHQBUFDzMDozovgFi360jVVLwS4qZnIiyP41CCnTPeQ4qReKwwvq\ni9LfkICX+Dw8aVaCfxZdWzGL2GUv1aTnK3TyYrZz/zyhjEQysNyWFd4eGIQd\njdCG\r\n=t6cz\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"1555d09a45cd5f88745a9d95746627cdfd22219f","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.0-canary.6_1605059797246_0.9244137908087473","host":"s3://npm-registry-packages"}},"0.25.0":{"name":"@blitzjs/file-pipeline","version":"0.25.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"7ab38b75e114adb60134f093945dc1e744c249f3","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.0.tgz","fileCount":27,"integrity":"sha512-mW9gc2ilX91m3e5ftDFzkrI75kUUxY34aUWfrRNksfZ8xC4CQNTqiccJngQyM+Tu5MNW8VSOd16MgVD/APsJQg==","signatures":[{"sig":"MEQCICAC8kBg4vQ7Zyqk7+KK1AMaBbXdEVfPLT3eDeGaPTjpAiBK49MWfVvHugslYKwHNERMP3jhGv9GAlbJI0Mlutv05g==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186560,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfrHTkCRA9TVsSAnZWagAAwkQP/AxG6SVgjhy53yMjAT1N\nt+ewoNMbo2nq/oiRxnevGP6FW5uuS2pTGnG8KTu4Q57NPNDAcxRjn11vNOpl\nutuhtuwg+xmIACygYIVOuDkww+srD2XRmPdQuMZa4MnYHZ5fT3gwn7YdS7ay\nyn1rKu3XAiMx27JKemSHiBAI5M8R7VxCelACdQ0RYIlbSWztS7yCOaguRKcc\nOIhAeAPyPjqHN++PvCSf/6RJRLxy1t5IWXD+WKj4mYjPIXXbeDmpG+X6rsTb\nnfnKKHunai0uuyok0y76jhUv8pPxg8TUvhBlesFwqmYDBN4x6ZNyktqJxrxe\nKSEInoOMmKo0REKumdj/L+gdl+eRGFjjX0uJdWxRSek9A+8JflNzrUjPLuw7\nnh14Jf8O5ONm3Xi3+9dnuVj0LAI9YaWcpAJWlPpuEOw6wYtobNxShtCxxfnJ\nyzs//bkCueISda+07aBhuUr3d9OggxJQJ6CPXMKUJg5AhYlDETHiVRmValhN\nZB1/0FEp/oDIrbRcclMJ2JWbYhsyJN3TPI5xpBiMq8RJ10ETnH8dyGjXbCXm\n4hWA9onzLc2CGKUcVwV/6yo6CdBS9aAVAafUXS7bLEPKRmGNaPGJlDWZwuof\n9MRFDlLpxvHdNw9PozfxGnADI0udDo9RSzoM6YuVRNbDJ4+EFLedmc+GmbL9\niqw5\r\n=/7vP\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"539f4572078d3dc670a9c4d89d2859db832ebf81","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.0_1605137635985_0.9718945322148749","host":"s3://npm-registry-packages"}},"0.25.1-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.25.1-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.1-canary.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"cade742197ecfe10dc078a371a9f51bb6abb0bf5","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.1-canary.0.tgz","fileCount":27,"integrity":"sha512-waHxWOreTpI/jHA6UNkmDl7WONgN+19AUiI4MhTrPhF54rbVV70nKQTaYQqaby+qAuZSIruBhXfF1IY2AANGug==","signatures":[{"sig":"MEUCIQDZzolI+VbUrKdH5F+u7DarhsBBiJRXKtv851rTLtzjxwIgLhcCxc0YMuTDJGnxhx+IaGud7nk+XiFMPR9CQdpnTQw=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186569,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJftItKCRA9TVsSAnZWagAAic8P/0oFWDI2zAm7xlz6yuTK\n17Rc2Zws/iGZWJUONV2LnM1hhewEBSy2fZQguQ8isWIph3dGv5i4TlrR1gp4\nzoZAlTWXCPeoCzfbCnusL95w8kkP2UQfFh1wED4f5vQvD/6d5vq5YG+1xmsf\nnsyCZGlxKR/Lx4lVS40FkcymG/iqKaaGy+Nh1+WxWizEp+l39CBOv94mhQAA\niE/78pacGOKAPMvz1YM34Y+jXzQW7vQMXtz1/owAZKFXYCwJsfeWz2cXiVSK\nPFcAoW6iyUWoqyICvLfQ+zxIIz0ftQbw5tJmXsksmeFPdmyo9IBG3KGtHn/i\nVT2atz7J9YXuD9R6Z9+a+SvDsxTmqHia0iKDIoW+03jwhmh/NvB+idxY1kPD\n8q1LdIoxeeMYAMRqDEl0wMtA4i3MxMjNLHiqDtoGEkSC+WQORfzpc/f0BR9o\nln30nx7Svt0FnyZg28wnYP5EIXii3uFf+ENfQ9LQ+V0HurvW59IYKfQep6FB\nLLU8wT3p4xLpD5XVpyaudS7FeY85caksVDTIgXqx7/rNB/NPsMIT4f5lHhAx\nglcQoaLKOIpL4dHSCVaYGMytq0HxU5lNvmsrGi7vSAyHCDH6NFk5K79bWmUT\nVULqvOg1XGgp9z3GrbzXEl9GSCBmgQIIq6lmEe0HTvcN2oAbu1STaOlAbzbE\nHzLV\r\n=fUe4\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"9819543f47cc03c52eaceb829bbdeff441aeb9e7","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.1-canary.0_1605667658535_0.6037767103325293","host":"s3://npm-registry-packages"}},"0.25.1-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.25.1-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.1-canary.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"9eedd146d3aee777325631425ca37272c8588020","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.1-canary.1.tgz","fileCount":27,"integrity":"sha512-ZFK4d/+SfB3pkV3VY/nViA/Q6tRKlemB6KFZywrKEs52Rjl/L8xFjU2z5ZDfgGMdXNBud0beHtixoVEQerRraQ==","signatures":[{"sig":"MEYCIQDT5XmEpYyAx/xFGQDepx+OqcvHRZztlt0I2G+HrD3JXwIhAPbPKSCaFLjk2GBzzx7//nY5ezfgf7S2suHKrNMYh8Oc","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186569,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJftdnrCRA9TVsSAnZWagAAK2kP/jGvS7vp1T5Sd4IAh4WZ\n3kisydsTKT45LeFLXkzWKf0JcaQooLYeto9FYnHvVnMRCy2jBONZrTGptXYF\n9q/EuDrGinjex5rht7X0tOAF2iwKRgRiJKauRijMwwvT5gs5Z2JFYZpvoicC\noRD4bfKdfEsPZJWwr2tZ3Rl5+zMNfX/ZmukxrPpP4k8Ew0tWsECS3h9Grvn4\n1x6xTOc81IyQe12Khpo+AsRsHGlpnrZ5j4H/LzqhnPe63pNGfhu9PXUDlPHK\nsTKjNzvYgJJpUYUlPsE8sN22Dcqe/HG0UxHd6lqcNmyF0+rJFWR2U3alCQHs\neC4qLhwA8jKuX0Nu8LI+sOyupIMRndhWC9+f1vDogiSyaKtzV3S1ssSTR9fq\n5UZ0xliijP8+DU6OB86zt7AIAsVmuHzjL8pQnWPEc5PUO9F3lxn0Yz21zgDL\nw5tJmuOCaQlDQF4FXutakFU8Afp7Y7S0U1g3Cxtz2SO+TaB/Fm/IWBROhtJ/\niKHxsRKGC98gXMC33cjOfuQImggTWNbDKjlkI8HXj+UagyFCMj0x1ltibYQ5\nqeY1TLw4sLG8y4s9zwyBGIYjPaDf4ekw4sEUxrpicAYrX3G680Tp0ZEzAaF3\nk8RbDv0CNAPeyvDjvgJ/+aewScvzeeh4jHD0Pyquugt/Bui8kyxWr7TrgMqx\nTTbJ\r\n=KSI0\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"ded2e2a546195efc88e670307631c44140f6ab6f","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.1-canary.1_1605753323040_0.8966199911017871","host":"s3://npm-registry-packages"}},"0.25.1-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.25.1-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.1-canary.2","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"39915297d510c8a46027d3c4a55bf2aee97a3837","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.1-canary.2.tgz","fileCount":27,"integrity":"sha512-JhGqcGw7Quen6ieUIUOxH92mCnD/033jpyjIcII95nm9bfzg7+DHQ+ASxnI0rlcyEKJ96wWva1e7RNHgjv2+4w==","signatures":[{"sig":"MEYCIQCiNrXWX+89sQ860fKf91Q5qOqhjR0RLtWWOFh/SYHchgIhAL+m7WHCGZNebmw0aRqQs0a4mPudIHck8Dp2X0MUh/e8","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186569,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfuGLwCRA9TVsSAnZWagAAq9MP/2eJSsP5NrOyHOWYAj1j\nbyHCqQy78kn3qle9/I7hZpuI1aOUTc/hv6RkPrwuRcM//KgJaW4yRclJMzhP\nYvMB98yTM+IVM66mjra7VyLV1xTG9JFveaX7wVwBfdBPga3o/fBVmoty6r03\nbMRdXh2kicpXYuObg+NJrnU3rnwX2H+PjCMWXrjiNHBkfkSqgnt8/Vi9Wv/y\nxFWv9+UfzDT7e8wwiMpiCv37duq2WPwNIt16jdy9Jvdv+VcP8JVzx/eT9dt8\nKs+/PX6uNYDCaJQCK1Sj0+WQ2d8uiArdtMDVLjFUPxvGrvZ8KMlVO96q6pyW\nRFZdju4OwCucowH4LKaU/MiAWakrIRv2F5aeXW3x6fAi1lBh4SMdrn2iwLHT\nUcgCgBDDDPoWzNyHavj9nXlVb9VXq4JDTrQfPf8YPixVnxifvLLDnnOG1UbA\n7816UfSqTdxcplVEhpSU4HAzNz1+HNce7glX6j406tFe4cTLPa+1z9d3yRb+\nQ+t2wDD96zMXEg27p0SM6h1EolgRn8U+smYfmiNdYtmCsgDYhK17+2nue3VH\n7pCP7Rx+PPwi5cM7KNSDff0N5GdfhgaSAXcuERa2UmEdtomDb//ecif3KGaI\n7SO5AmWvzi04dDiADEsDTHbMrnVNN4O/3/qfh8Udb1qCN3OVYTz9T4gAvZPm\njHDE\r\n=kR+P\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"f655410167b7b7c4c2f534e2cfc7acaa60e2716d","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.1-canary.2_1605919471499_0.861237681592987","host":"s3://npm-registry-packages"}},"0.25.1-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.25.1-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.25.1-canary.3","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"916848f5c4e36481704986eac2a03523774abfc5","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.25.1-canary.3.tgz","fileCount":27,"integrity":"sha512-tKajzFqxJFDYP2gHpKimR8Xo4HZM78gSIZwZgf70YxRg8TUp7sGJFN+JTagBSgswiF3ENPuzhKLq2dbU0ozFBA==","signatures":[{"sig":"MEYCIQD0IhiCrh7wLg/yrT106hiOsz2/g0sIaSXFc/cKkmTIyAIhALMsiX+pM6kxLat8aOiFS0qhCakWxzTDHJCWCKAQ+V7O","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186804,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfuHwMCRA9TVsSAnZWagAAxgIP/RH6uhUhICuqOBJWPu2q\nW5s2xw4iUDUYsVfsiZkZvaZszlQTjw6FZeN9HxUxxmBphsNme1bmZNObbdwA\nfvcn6AWSM3kjIWZMMuc++CiH6S/EQwxIqYt9f/eJvH5lFJI98kuoSDqtz04z\na54qUsnu/tBpqXu0ddCMpgOFiUz9NMQc7Ou/cmsYRdeNG4Rn9OcbWZg01tyb\n0Hqn5Vmz+kYG4MyzK4ggtN9rXyBz5ywKuEBnhTvXtpdt34IM3IaNFT5Y2TAE\n57CnRkrB3i+SJObnIi3x7bCx88b2nriJYGaJnTmLxQV/eNEzezJW5+Ohj7hq\nC4JaYUFcynltXec397KMf2R39vI3WUKz0KJ006UfBjymUBx3FO/rOFeJ4KeP\nSrrdH0um3fFpK2jhByDmscrHpAKq6r22iTB0LinhDbIITzRacGPXeXeynXVW\nYCfpr+GCbDTLYe6ROlApBUBafsrJco+gaqMzWFb7wsk0aLXVxn/smjkrEDnH\nTgossjdNptDIejgHE5M2kdfznu7FHcfWKKMvcYBJDXA3Htdqk145xjFrAa1v\nYFR0y8qKg9JjPNqHbcEXGCVpusXT4aZpaWHe1Ar9XGGv8xSGFONa3mC9IKtL\ng8mJDFibujq6BNuX7sEnlhnJn/wwwf4gvxFz+aJ0JUXVNIA14CrTQ7zF0utE\nRufD\r\n=pj7j\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"3ed4844d879394ecebccbc13db659b9fb1aad38d","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.25.1-canary.3_1605925899635_0.23656752761472366","host":"s3://npm-registry-packages"}},"0.26.0":{"name":"@blitzjs/file-pipeline","version":"0.26.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.26.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"66719378fe22a19fb652ee365ec3cbe21b7a57b4","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.26.0.tgz","fileCount":27,"integrity":"sha512-ZOzD65eTM840dSZFS62gu+Q7CjCJ7LpvdHOHL1drBaUYReXk5Lu97cofDseql1JoE/xJatPk9+1/51jrlMABpQ==","signatures":[{"sig":"MEYCIQDcZjh1QHI69JDtlknDgYgkee6SGMr8hTGQpJxObA0mUQIhAN/9UWmZj+hvj536CXaPkMTcrFnk/fG1MtRykL650gpz","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":186795,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfvS43CRA9TVsSAnZWagAAM6MP/3AEOneyX2Hakf8Dv660\nNYQYj+B+RTXEem8t8pfrwUxGg0ksr8zPnCyOMRf7XLGUxOQnXaOuoVEhbMmx\nOcJgbntYvYa4H7u1yD/UvGym9gNhQ/dpn6oO0ku15QmlUcTf78z9eCvm70ph\n2sRoPqkNROeNBJ4+UNRZStkP/sRrIlVtiDY2vf6uJfW1WYPRgcoEHQP12WwS\nMgsaakN1aDvVkh5S9cD+t2Q2Q0nTUwXXpLw2FhsX5nGDf/h4865B/Ry8xF3o\n75b49tFk3G0pOMZgsXDG9etiAUQuwUbWlK38M10Yt+LQiBTrBUkFKXmkjlmv\nhKuyYppGGVe0GUfZV3HQCQv+Art0fLFfyBMSLNx/qaX9/wkB1d1dZhSIerd6\nFP3LkRFegYBkJBqXJabUPS2UlQ41g7wqaPwfz84DjHCmXDJ6pH8ZYkgMxcQJ\niEUO3EFCFcxKYUx8JHH+CYp0AiL7yRldr6lNNabF1Xv6J4ZPWWNoe4yQ8JTl\nQLMtrkfJ7WJDqbyvpVUZZYKaFg6pQUkBkMZ4eyeT1wF1LUPyEyo9mwsfHOZ3\n6gs0PbMNiMhdcjER60IBH8TQqeaU1DsFYWVtzLrsDXtXHS4whxRIz0DnYHob\nx0SYLf6+X4r/NBONAZ2UsQAx+K5TWJZbL8u1bATKIPql0Lvblz5D/cBN/cWz\nvmzy\r\n=E/ea\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"394d7df0947eda9d395240e0b8a4030f2d23f05b","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.26.0_1606233655027_0.05617472562757664","host":"s3://npm-registry-packages"}},"0.26.1":{"name":"@blitzjs/file-pipeline","version":"0.26.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.26.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"f3b593ed52909891b2e99c983dcf8fd344857075","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.26.1.tgz","fileCount":27,"integrity":"sha512-Hhnz/sAJQH2SkshxxhPRrZz2YeXrSN7LLLjkfza8p6Iv79DCthuWQ9huq0499yUEm5uudJ/tm/LfWWJ4i7MOfw==","signatures":[{"sig":"MEUCIQDg/Jak1upP3pNOYUaLlF3/ksxdo01WmY38XppkV5s0lAIgIHRkY0MV4wvyBQiWDGLfo0MKmJPTE5qamBsYbgd/BvM=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":190835,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfwTXeCRA9TVsSAnZWagAAy2EP/3/I7074xMn2bqqjUCFN\n2FPdQ6qJnQg+UYif7KNdd9jdR+kkxfqbwcep0XRKQ2EARN6AVTRtI6hdfUzv\nj/ds+CQIwMCs6PI8nh+h0t5hnidOD7URf+0jwddqjYfST8Fh3LPJDESuFhbY\nV57gl8tKwTGFlUAI7j5ZFALCib9PHQdZGhqjI5cVdajGCIHZhGKR0sGeYf9+\nSFOlUrBx537FPq7ye7FP9yqxYRSRRBps72O54WGsc7uY5Yjq7x8U3PWAk+3T\nYTkHMD7hXW0ooPXhG0FYStZ5QFLNqU+5yrD0YM/jzd0oCl5S26RuTEL+uRJi\n9mSrgKg3mXDDQVeHKP52H+EUYenI/GgQ/i0LLo5csOk5qmGOLRvOqOBGw/WY\nPazQMV4Bp2Jomb1ndJ5jRP3oo+6l6nxUleT0kf1B8MvVc5EEyku92WrFImVP\nW8/dl/DZa497IGIvkhLHcCquwlzkLlERtfFvlugZsQKu3J0nmZJ0MfVg81F7\nW0cPUiQheXFW0Fx1+V4LGTNiwl13oYygaZU45FveNPdXArqhMeVAmFsa9ft7\nCvQiUERyt56dRQhHC6rX6/bKZuPLppy0yZnQRj4FE7F/Zp2P+JSdmLF9FpIh\n0+F4dWqxCpQDAA20b8+d8DWcAxWOz8Gy8kq2nU6+W8IC0XGW7u+XOx9zWoLw\n9bko\r\n=FuIw\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"0b5ca09c7f7ad693aeee69449fe8790447ca8292","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.26.1_1606497758065_0.31111649734104385","host":"s3://npm-registry-packages"}},"0.26.2":{"name":"@blitzjs/file-pipeline","version":"0.26.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.26.2","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"a036782021628639fdc11ddfabe9fb2cd2720517","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.26.2.tgz","fileCount":27,"integrity":"sha512-pq6RA3IJNJidYRux5kjsKZ/THfanx10zEEnzuiFW5N5PGjmMSS9iTRZiyeF3d95vouTW4seK6HPwbVo4X+TDKg==","signatures":[{"sig":"MEQCIApj4L+HNnFPoNSaEtxp20lIE+tHfrLM07Bj0OIufwplAiB4wehjHJrHBj75x8ma7/nJReGiil+JlRI3+PQqwBkyWA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":189679,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfwWPHCRA9TVsSAnZWagAASwsQAIZWQArZFYFZGx6QhtLC\nqkoDuuyyxul0ZcNKpTRu1QjnB0qNFopOSELolOZflCnwVs8sJd8Ji36t0sAp\ncHYAvK79K1EfUOFw53pv9UbyyzTLbUa5soCIyn/S8HsulZjR8erZIT1oJnqe\nrsUyoBL13XeO8as2qnEmZ9hunR36scrnaYohEpNH2lQyTjodnkK95QULWv9R\n4aYVVbqjVyX8BFLW5hGQhTwpmfODGDRzGReMt2gUJdxBse86paYuST5q8tLT\npEbApjH6vT87Evh2hyajdyi3GvhKl8S1rnsM7bfwRy5U48kBk4OuDX4vMusx\nVrblA1/WYbcYJYpicLdq5Jwij39jzS1/1BiiSkeL1o8AFWbiKQknwU4Dcd31\nCRe8nJFbOvBuigQMxVoz9ZjOorUPzPEeMEmdfnh76hVvUAUpv+LEPdcOjopQ\nm8vaIQAU21GJEycf7mkf+vWxKp4GB7QHhznJzyTDxjNR0bpEXD/2AsuFlPJb\nCh4uBvM4vnlmgA8QDcAPyZs/mUXS2T2/DMRy2R3Pc5rdlhZCdJyuUILpCxDe\n0jxjq/rJe50YGECop6uG+Zi75yqn4Vg3OHm9FsMvB2bsk8fZv9eE76yegx2x\nEECcDi8DgO6vtd/1g7GWj5KsWhE86K/Maqu8bifgC8Z0YtISHIvMqV+RO/gf\nyL0d\r\n=df/1\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"05431ec7b5be7bf0f2d63d6aacf47606995aabe9","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","test:watch":"tsdx test --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.26.2_1606509511171_0.3464130738079616","host":"s3://npm-registry-packages"}},"0.27.0-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.27.0-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.27.0-canary.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"239a2252bc390f5e1c75381bba331b987e5855c3","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.27.0-canary.0.tgz","fileCount":27,"integrity":"sha512-wbWsZbH2rcI38yYzbFYwdeeU6diLIfIcMS2GNxTaJ3njedOsbWjmVKuOYKFQdhMzkkdSNzHquL+PGQGWR+CcXw==","signatures":[{"sig":"MEUCIB8MpJHf1bpEBK2Ies5U0+ktc4LSdR9i4S3HG6Ow+624AiEAmMK8quFJNtbQKlObVZZNjqxCQC9ump/2plf5O5zH4Ww=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":189833,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfwpcrCRA9TVsSAnZWagAAkP4P/1dftQF2IA7XnVLN2r7M\nPTtaoBmJviFv33VZLwxOi/xEP7Y2TI5H6rwpb2o/09BX3rUUhCqrbpQVQrFq\nTwYr8zLTuHcs3sBpkQLHDYjfA43Ie2MSm9J91IliN5M5tuRi1JvtV3rjyL/R\nTpgHl0PWD3DHLbsqbPN/lFiSJBC22/o5WX4NdyA06AlIvbxbzo4n8wWJz+gZ\nrMdQAWeeYH7VOVKBCrifD1GuGMVXDHA5la6U8FRLibZs4gTk2sp0erLzyw3k\nc39ET0qkB7QDCx5U2EeOCEF/3luurJ1IHljoNXItFoeKoXoIIdeq+XoURGOa\nm9a0CM0XhDhX7hwtwh34wdpZCTPFF8UN8cGzZ599bWd2zW/8lgDAWkuD5GvJ\nQjy0Aax6av+jc5YxZ/bKZxANRiaCakeGPNOG7Of5eaILEs3Z+wCrDyeXNI/L\nZDyDS7HBA0rmC0Yiz8US1NJBqE4NWNLquSjov7fSypuh4hDfRwc0jgP4EQn+\n/axIsZFtvMF9q4l4B4MIIcsPcN6RQ1caRKjLEOzlksOl4n9zdBUjKopK0b5J\nzE22oYMZDZncuCu5w1WB4pvIOQls2vFUKp/iHaySIe6q9bMDNUGmVYNryAjA\n4qpQVON/B6I8r34RpJdf2iYhQELLinsDnSnatJEbRNHObFIryEdXqnNAaLkP\nJtbC\r\n=BHNw\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"85c2aa280d7f310236aaa8ff15f1a5c61a31e92a","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","prebuild":"yarn predev","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.27.0-canary.0_1606588203094_0.2936346613507428","host":"s3://npm-registry-packages"}},"0.27.0-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.27.0-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.27.0-canary.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"062a109d4732f5b0516ddeb25088e5a8c9e34526","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.27.0-canary.1.tgz","fileCount":27,"integrity":"sha512-SX0HVvTYYeOOnBvJQXjbu+EKMUn6Cu0EvpJ03HHopyu+4XGSXtJZVzpCeU8mF6n6RYdTLzdXWH8KZ7sRgK/suw==","signatures":[{"sig":"MEQCIFRLzq57HbJVc9DB77GPgx8Go/bJpFcsEkM4KQ4ppn3vAiBY6O9VE6ugAQQ2CroPoVKt+gV94GaLlgexMOvFmD+SRw==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":189833,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfxoT/CRA9TVsSAnZWagAATIoQAJpF6XQya1LiBk/3LWx3\nxfD9OUS93iG9u6BkhljaCdAvM8kfPM7iBfoYqB30cxWBu6KU62XDbU5gi703\naaZSc4JcRHlKNP2NykJgJsVPkD7haYZxJFvzDe0dM+Q7HGPFcLpBENuKQ0oZ\nftdJFHO0U3lSeW1/MS5qEn+QcZy2WptOPSZfvsiXnTmZ0sS2ViIBZpimlOZh\nM33zpm0wmWyvmB2jiiHGP2NcDdrJvZPUq6SLWKCv3nmkEoiIxN1EY+x6HF8W\nXclgUgjw5MooS492YH6tNAxd8VEJiV5RFJPgVQLRUZoaHn/lbZYx05NZzzIF\nAyJqYUOSwmrg1lNFRHyIyvSki7uhdKg+nF14B6OOkpGCPBMXKLFa/rS9NBzo\nudJgwG9Lz+qd5iGSHNV+KZL52bbxdDRDXDDLAZCKCgOrj/GqfTIbeAzMYGB3\njTG2mHeoaPaOC/bf6IaVz9sofyteVEu5FBNVjCQUy6kidaKbHOflHqUTN+sg\n4hCz2zQPFswyFM3m+d2nDKNqW14WUhPRJpdczGSFo4oQwtl+GKYDCXTF7yGm\nYOqUQ6Z1tKqgg3052JVsQ0q32eGYNDk+FyKMejc2p0G9AUo9ms71M9xL1EtY\nyiBMPaB6zQMnlxtYr/v10syVpVqU6TN73b4iUNDht+n/4io4ud1Da10/UROR\nGsj4\r\n=akGc\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"6954f448ad44ef43fac43eef8a14a825512206ea","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","prebuild":"yarn predev","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.27.0-canary.1_1606845694928_0.42178299394867813","host":"s3://npm-registry-packages"}},"0.27.0-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.27.0-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.27.0-canary.2","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"d678e1e903f8975c57b50fa92a165a9587cb7a0e","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.27.0-canary.2.tgz","fileCount":27,"integrity":"sha512-lRWFwY9ZksvCoPj03Oilo/7b/BZajUI92nkiKImtp6B4AEf8VEbIS+6Ia5198M1rmO312u1LSdZQ3As+vvLlKA==","signatures":[{"sig":"MEUCIDMZ5x7iAbbZw6fJubZaU93wwFyu4TZFa8JQVtHjOhRlAiEA5VJG5WJ3qDKM+yzzu1XuYKe+81MlA+TvkNkpcJLwuwE=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":189845,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfyXlpCRA9TVsSAnZWagAAdvQP/1cV+RG6V+C0jg5mvfkf\nyVqv/7+O/Exosx5AtWBSsUpBHte4OwxtD7Ud+xNIHcCsW8vQ92yA9DwQRiP5\nmW6xYP0UwmYdQQfPKbw1GjYEorxXX1SFNLgOErFzk09h2N7wpCoYAJPLRtOy\nQAGSPab9aYY5niwG7C2I/ntuadl9FMuT8a6ZG2SnBKsxMdwHgOMp5WN+RdmR\nwHhCPmVvJYRlJdTPhbZ/HpX854k4SDauoOlfuIscnWUxXGFvbKVHC9CDNlIG\ntomRvcl/+EFAPI7E8Of0tarwZCWLfphpvB/36MUz1W9gvV+zzJtnLhVTTXbK\nJ6yhajCnzPBbs7j/vJanRFpTOzT8wTPW6pe9CEjqPTsZ9viE1EMpmFUVuwbD\n6NOm4rtJ3+V36x6BK1cAjQ9kncKv194vesEGSza+a2PSd3H0dycj25JxFjZF\nfwW+yG4nBjFcR4YtXDAqL4RmMcy8KiPTxN97q/y7cXVGy6JUVbqkJXCG1sha\nctV1s8V87mRMnqEcFL8hq1b2ji8hS8Lufp7/1WmxrBvDqO7G/iiLhS5I+2e8\nl/OMc1knsCOKNJ27A4CMDWRmgNema2z/HHQ7+hVej+X5dwEdMqJDuZcNOYa+\nyb/HMH+v4lB8HneENutV7vjBccYvAOyMebJv3InrBdhF9GieIRAuHt+nSuzV\nd031\r\n=SH4z\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"1d3ca101647b0e10159121d0f1611605c242cc98","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.27.0-canary.2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.27.0-canary.2_1607039337293_0.18995536973620597","host":"s3://npm-registry-packages"}},"0.27.0-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.27.0-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.27.0-canary.3","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"93179a7ff3b538752e3d0d6e286d0cfbab2aab5f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.27.0-canary.3.tgz","fileCount":27,"integrity":"sha512-imcJZ8B4dPCzIfdECswNx5YAEKPLq3lrN5M4z+HWAGbMoVn9iTZ7FcSu73tBze375Zp4hPwEBIKTm6sfZFz3fQ==","signatures":[{"sig":"MEUCICVAUDI0VT4yqy/1VLNAPg3KuPInw7CDCsoznF8zam1vAiEAlsPEXAXmVFHC1aCt4CgvlLv6dtyGB1NregelAV2NVSI=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":189845,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfyXuTCRA9TVsSAnZWagAAHYAP/1ctDaQxic/ALFmJkuX8\nuoiMme0KOekWe61Po5r7V4lnZcDGd1ViVLU+oNP7OllfM/+zkZQt55lahIWc\ngfAjc1tknqWaimOSNutGPdq2Inq4Cg+WrkY2deer145bekHk8Yn/1HTKkLzq\nMgX14YN65qIuep7nLuKqVX1hdPAwO6XvjRtPP/Os9hN9UHrhLtAldDj4RUOx\ndAh30vQzu8JOZrdcNvSRNnTigigYwommqWD0pQRv0MKZVNCkAUZpdg4WwzLY\nc2Wg5w5cWOd+kikJ3ucTGQewhXpmcMjx6SimehXaBox14Py1a0yG/ohG7/Af\nJG4P91ymO21CVij44ec5E9HHONPME1q+uzVnqFRH9NCYS0RWSGCAe39yDImP\n6ZIJSp7U9jp7CeM7G2x1IlSuIZuvCTCoKxPW+bA++eBbRN4+iLEqhY1rksDJ\nS+ZC4f8u3FLBpZdPkbsIwg6rHFz7lZrH1+8cIacT3Hv/0+W5iSDe/EmHNNYm\nsflnzjPnf5FIo8NqG/LIbne0qlF4UUFUZstBSYJOSq6lq2LLtBn4nFI4sDEV\nifYR8QzRVBxZzZMYTrhEGrs8E6LJtADeejxvfChfZBSu44039qudVoo4rB29\njo4djpKuNEuY6sFGLtTw1fOUNu3JnWZVxS+bRxuPnPFJ04mqir78el8pN2+f\nw8m9\r\n=dKSt\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"74083f704c8b51fe162ff237eb0858370f3b42a6","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.27.0-canary.3","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.27.0-canary.3_1607039891209_0.496057227775206","host":"s3://npm-registry-packages"}},"0.27.0":{"name":"@blitzjs/file-pipeline","version":"0.27.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.27.0","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"0acb6c52e63b2d76a2ce0e103834763c91231117","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.27.0.tgz","fileCount":27,"integrity":"sha512-YrkLKoVFBqfQe8uMPA1MWr9iU/YV8r5AVu0CAaUvM3m7G9pE9peQrbOUw0CGYcpJBczR2SbRy5wuu3XzRJ32AA==","signatures":[{"sig":"MEUCIQDBOXMbmtSQ8AhRHYsbFmi5bOACOlt0lwFDdLWZF9N58AIgN9CMRpFN+a8dCT/Sprj293unIV1N8Zmr+MsdvHaztoI=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":189827,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfzCDWCRA9TVsSAnZWagAACQ0P/RIIA6z1bTYB1YDJyJoa\ndHCPoiVWArwN6Tbht7sLhzFQz+1zmLVowMJEqVFPJ1ReCP7KJ+TwXReC/RXa\nid05PWuDqHxfabx1J+x9S/XFLqwQbSXvs10kIwy4ZtxweJZY5J9EyhjBQwDQ\nlIfsgcaHAJqMBOoDqBk8QG3iE90k9Xjm0Alt0w7UR079KXy/YhyBE8gUg9/W\nrf2ggODRPBGxeuD/9s9oJujUx6ABZoc5GU0x8hoKsh1RRu7RBvI93PWNG2bW\nCt6sKw5UDJJEr/1GoehS1HLLouM4IAIlgoBWIMU0LUjPQSACmYwr3ZC48WOi\n2/0GieIpn5ERdFI53YRCKNRxlB8T3J8myINAO1OhxRCUQBIKhQTSWyJ8+PuW\nzvn0ObYfYZ1v+/uht/h4i3jsU0HLwbbTBurYH6qd/pXThWzMh4AUUYM2B5WB\niXOuUP5Axdat/CJb1dDo29cKn1fwSfrwpep9+o9cUCiowVEe5sDigPFJx7Kc\nj/XvLlrnuyD2PZrD8hkxBJKzn25pgYtKaoHgy2idz26A9XvgWqdXfOcJG0ql\nm6gaih3xN9h5CaJmEzktH0bRob/iBn0keth5gIyrpgzMG9CBZehhI+qPlHqM\nJrc+Xv+8DwaXo+Ccp/kDCC5Qc7mWAb3KjZAUA/gb1dCCA5V2TqqMjOZuDZXW\nBr3Q\r\n=8jrX\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"688b05df78edfa3b47a7fb091e22df8ee9d622e7","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.27.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.27.0_1607213269577_0.622046189696551","host":"s3://npm-registry-packages"}},"0.27.1":{"name":"@blitzjs/file-pipeline","version":"0.27.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.27.1","maintainers":[{"name":"ryardley","email":"contact@rudiyardley.com"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"7a438339c8ac8daf3dab8c38986df1a92b8378ee","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.27.1.tgz","fileCount":27,"integrity":"sha512-Cc04h4Zn4JC7h7yl1yD8NZjTtXsBNv45ZkFvjM9lIFA34Us8hlOd2jAExhxmKkDOCNyy3xWpXDBBTANNvgFFYQ==","signatures":[{"sig":"MEYCIQDASzle/JjYvrCpsBBAQTOnDfCd+sToNLfkIDvNT9j44AIhAKTBFxpdYhOCG7F+560V14tFyT31pwRErKDUxN9ovjFR","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":189827,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf0rs/CRA9TVsSAnZWagAAfysP/RBl7BMcQAoTRvv/e/Cx\nsZpq/ECri7q9u3Z1+soBxEelagN4LpsRVLqg2GrMx4jDIhV5zPEu53lWZH1Z\nlxy18vIhDG+tSmfWZz9XlxFDPUAj4XrpZZJUT/VUSiJGeLVqJbc9R4zzPEAg\nIncgjpPnraF2uSj9odEBQINUyZ3fwubbxCRXbvr5SccoFYB1Khd71BEgcOJL\nCDudkmjog/6nn8PrzuiPWLaEytqve4sQVf0iIMIdcgTBFoTEQEcU29sOg+Ik\nHzzYAs4ZnHwY+YvPNNNYEBqBAd/aetnPLt+qmKvQDn2BNDh3NroaHQkK9zl4\nxM2PxQR+A5d6SLTMr1M4SUCO+3DXHqkhCPYium/334AK4KHYTeXl8Z0sRuYm\niSl7IHcQWZpNttaQBvsNElbhwboJxj+FXAce2Pi+zsjqhUpIXaWrg6NsmWVO\n/aarCd8gfrMZpVxvoF0FPdqPbsxIQ+EMs/Pb8lQZOEmbMauJ63j8yV9BA+j0\n/huLZlWkxwXqHbAURsWMeTgNmXyvSiGz26+07tqUDcQXrYqxdtNR6LcuurTJ\nSyJbOMUjUSDjyYkKh2Ay4yBJyiIGU3j8ntv473MJIyWJrhvWBCw80Qg7Ck/Y\nNtWY5MNCq0ExI8W095T3Cca0w3wFkfpPD1xlTM2BPvueOTNl9+vUixcfbveQ\nF3UH\r\n=OSfG\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"b2794b6cd535cc627165be76c05c17ee6ef34687","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.27.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.27.1_1607646015324_0.5676377513558528","host":"s3://npm-registry-packages"}},"0.28.0-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.28.0-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.28.0-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"e81e006111fe754cf5934bfd6fc0d29922f7cec5","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.28.0-canary.0.tgz","fileCount":28,"integrity":"sha512-muSqtMSXxYkAg7cT5u5alRL+y39nJW1V+Oo/md4gMNmz831XPpwdcIc/ujbex3/YdqqjBe+SdmBhzAEwi6R0pg==","signatures":[{"sig":"MEUCIFlidzt9ttMhkDqzAm5b+PgqEka1l96p4Ipq2qVCVG8LAiEAu2r1Jyikf3LH6o5yJckJLbtlktJo/tWGCAoWQ5saXhY=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":219956,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf28yCCRA9TVsSAnZWagAA6VIQAIXHTINqkEiORDHAWezL\nMybOrzpAo1PLJ/aPSjbZWBzTMSaWIhKtHA+yHeo4+96E8wZCn1IaEJf/lNn8\nn0ayBvnK5Mmvhl7PwpSOMTSyVnpiHzgpx/RcjE9ctFM0vXTq9lKPJcamytT4\n5Wz3tZGn+3zOrSoC0xw/Jpnk1xKPTvVWavWj3E56BKZIeF3f24+XcOnVfx0s\n7vWd6YQL2+s6Y16K96B9Qexo4WvCI576FN8SktbLWkQguOWxB9fwitBmlpHU\nmKOQ9lhuHcsT7eXpJLPNewxW+pmb/Dy+Jimhz+Ahwo8/9Avn2h0+yCX4DEhh\n1P3jjrZ9tKQaFxy39HpWFbsHsEA0dIOqNkvsV2ilZswc0bWuZvhq7bJr1W4n\nDdWvKVA4koZMK1CHQz5wSq+mIED3VTMOaafXTlNR05B9dOVCNT2WiiUU++xC\ncqvkA0n0oh6/9SFEbGa7pUdaI6C3MUpynE3w6dPkbP1Qcu54mHzCT06r5RlP\n4vIzEjFdfP8aODt/S8jK3lWcdYVBx7Ljf1StWrGGIIvAz1UwXCYxaLyPx3gJ\nRe/SUnZL+yL+WK/p4pyFL9uA4N3EeBXGyfL2QFag/0HAvFCz1iglFON7KEEu\nss0vcOGagXAdtxJHgaxlzsPYHc4DtT+WKQw7LvRfY6VEfBd61x6jjH22BxDO\nYgM2\r\n=79DT\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"d9c529943630de96b394d86b2ae5dcaedce787db","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"ora":"4.0.4","pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","slash":"3.0.0","vinyl":"2.2.0","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.2","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.28.0-canary.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.28.0-canary.0_1608240257696_0.8706798123130612","host":"s3://npm-registry-packages"}},"0.28.0-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.28.0-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.28.0-canary.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"fbb17ae5323d79b34a7cd81c597570979f7ac483","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.28.0-canary.1.tgz","fileCount":28,"integrity":"sha512-BXnn+MEXskzgyNAaNCKUI5e/q8zRbBeRTMKB3hvBvuLRzFaOzTpdA14TtldVkPZ3nJ6atZH523TC3ko+K7CI4w==","signatures":[{"sig":"MEUCIQC7fA0mEdRYa+5SQ5/Vuey6xhvGu8wyj5ppoXd9VMuxEQIgYY7aljEerJmQ4sA5EZhrWHjhzX5VqbAmsevYcjenLEs=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":219914,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf45jYCRA9TVsSAnZWagAAOqcP/3UETm3zheMqZ/IplG8H\nNPckTRjBLWdWT6X365vsTLo3B89nEksnLT361DR1SQcFFTELKnv5CVY8ng9s\n+uHoaJGpx93irYXdMdKnc+d4E/gTiPqiGSvuB/JjOdBW7nZZJN4o+vb6NTcX\nBz6DtVbSHtClZyEW5U+3buCjGWJHT6+2u686Kwc1q0fJO3mP09cWlO7i3HxM\nivGU9C/4lt/fw59ox+eFBSsfM9LaLGGkfukgV6UP7P7bSKOq+ysDnk8g1a0f\neGhP1V1wtj61UlNXF0gos22XIzBCg3gbjac0rtdgmoX97w0ffzKeBdOeNAJd\n1MQY+Nhz6agQmjr4wKMG/iW1GpDy9qsXHLaMHmedXzlS32XNJ6guYXoOBMwJ\n9NpikMsdUq8fTHK8r+UDRFcA2s8roLv+gYG5BvGr6N0c7vFDH2PgT+SD/dYN\nrj3aQLcb0CmNeYE5NsOtYrQt+Cf2Rln0kWpuh595pnAO5cxa311tJLXNUzOx\nGBsmOpcz9ssXL0yG0TldcL3j6RNyTzTxUq7F3KyWzWdpQEAZjJa0oR8LRwSg\naHetn2qTaPTOnuNcHdo7dPSa99LoTO+xjRXb6CO59UNURs0tv8evnnvNOKUM\nYAF9UufbdrRwL2O6vqMN9LAf76zWyeQbf6OyltlYyNhvyDA/AU0IYK06GlDr\nVnht\r\n=jq5c\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"cb80788734672c7b350e9ef2253b9c235e32a198","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.16.1+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.16.1","dependencies":{"pump":"3.0.0","chalk":"4.0.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"9.0.0","through2":"3.0.1","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.28.0-canary.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.28.0-canary.1_1608751320455_0.43121568705744595","host":"s3://npm-registry-packages"}},"0.28.0-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.28.0-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.28.0-canary.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"fa95a82ad3c99eadb4d758018181f0104a35259d","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.28.0-canary.2.tgz","fileCount":28,"integrity":"sha512-W0L6enyXjLD2eP+kX4LsMuZB52hA05GLrvFtgoegdlAgKyQ2b0sSpw8SgGn8T3uKpX7LXJv+hoTtZOGHwpD7UA==","signatures":[{"sig":"MEUCIGNAVCFswAQv24drElEW84pj4t6Z8+vuFkIHHB1OrO5mAiEA4j0qVf4pZtu9qPnW8y3zagx/YK827LsKQ6+Wm57eFQI=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415276,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf76P+CRA9TVsSAnZWagAAsoQP+QDUcjrfx6bFjox3YnFX\nx9MjESe7vFBi9veRwQumghCmFpIpTxQUAXA8GtNvS1R6IZAv6IzffpfDKJP5\nBnzcuDVChQdc+zA9dnQKIaR5PgAKRPOYvugBfCyJH2Msm3LArcBLKws/is0q\n8VB0aIPDBtwBiW2FUeRG0NXbMlSYGk88ADAph7hBYUEiIJk2eSC8TQBvE0gD\np0AbIBQHdWDEBDkVqOewfkaeTBScNBRiZYJMQ/zgyCTCHX/dXdPZzNpTAEtV\neMOacV4VqVGBep6b/1ifUYM4J5/+mLfcq3g2nhpV68S8My9vGOM5zdTv8vcu\n+WmXTrMxkm//d7D8wBOXjDMQ17eyBTJcqyaehqB3ByzrWbdox9P1gC97Ywjo\n/Y3FJCvayP83XDD3nGsZrj/raCCfQwW6/zy7CAa1zOW1nXlzW88rW+NiLP6T\nAhLIzGIk/yFcy4OmCUeBQZkiaCr7USKShL+Fk9kZbBYBD1rBI0ZATKvtHTie\nS2v/ZvVh4rI1KHY9Gjr1AZ390GJzgrs6u0fVpN3XXQ4RrgPSzMYBVByNh0BF\nfTI+CSaXfYox6YtAB6/rFI1nnYY4K78Zr3epF102LtGf21PC63n7nnWY6eYs\n0W/LbpkbRLsTuMI9JprKaVLFA5np6Vb0ezOTJJGtfTKLKNxciEWS1ZtlzzEx\n1wj0\r\n=r845\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"9be0d74816cb6b15f69033e5e8dcaf07336b2465","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.28.0-canary.2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.28.0-canary.2_1609540606249_0.6741267201181527","host":"s3://npm-registry-packages"}},"0.28.0":{"name":"@blitzjs/file-pipeline","version":"0.28.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.28.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6af517946f07d17779eccc6d0490ada6293b0b6e","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.28.0.tgz","fileCount":28,"integrity":"sha512-s+8wOkE2uA0lS1NTVM4BX+rJiIXzARkfRgrGx1zNwM6YM5HXKKq6zRYk/Hj4BuIOBA8msia3qkb3V18t63mX0Q==","signatures":[{"sig":"MEUCIEDcqfZmwb3aNL4GTVoLN8H/yvxAVsCND0t+E+9rxzPwAiEAisvrkqCOYCC46syUx82n8n/dVxDS0ISmS5r9/WC6Rk8=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415258,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf83iWCRA9TVsSAnZWagAAHHwQAJ2f9G6ds1xgoG7UO9ZO\n7bvktAHqhyuSDq6xWZmFLONR6cULunhMBiLHf44l8p2tgTYYn9sliFHASokq\nKMkcRYpa8dY3sNcYSnRvZ+hRwZLDzuChtUscnp1TdVncz4YhBLv8XfNBK3zP\nci6YpcsIBrgL0sXsKni39F17KFdOwh3eahxKnSdQtSv90pUSpZEsxtQBTkPq\noVFX9vrHJyN6V5xxPK5hhkhC9FGvOi9cbdIBQhDhcMqkQxKxcjIQTmvEK58k\nyeHC/VR1OtWbxccpjUeX2HQULqV7H/SEa+Qt5bxBLR2M3bNmGKQ0p6mGSuy6\nJA883h8yiLORDsz+6gCrrjodfPRXgu3Z7hl28newWGqpoQI8+o958QODjkzO\nhdeNsX0i5pM2098BN3lQ4hV2H4YwaEEot+F5bkcc5ugaFaN8i1R2PWjiTuZM\nTnoSF4HlDYdJTwvNQyi9KaC0uLz5tc/ZES/9LhhnGG1+7/cnja9ZkrptUhI7\n8AcmD43CUr4jCMdECiglv5nUjNS3WonrJPE8BxSmMk35mmBFvAi95kMfd0TE\ndhX9MrdUu4RP+K55YECtaL4wTSVsPvuLb+YVahtNgBLRqKloCi68Citz5sL2\n/zF/+CVR9Mt2v9XD46r8Y+w33K2IYQbalXAsCKcold/Bzd9qoOinUAZas0bv\njIuR\r\n=aleg\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"2284dd8621367889e3c0c5c187c1ee822dfe80c1","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.28.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.28.0_1609791637225_0.548576838884768","host":"s3://npm-registry-packages"}},"0.29.0-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.29.0-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.0-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"b40e8440237dea089b9207ea032ab1c85a2dbd74","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.0-canary.0.tgz","fileCount":28,"integrity":"sha512-Qjy4W2vmIVuX12SFAi7YUJPpguW9FgrWsXJcANOuhEPW8eaGA6fK9HZTbc1k0pDUswQJpXWw984I7yI0Giez8w==","signatures":[{"sig":"MEUCICZHwCkprFvZh3oz5xt9jX38uE3c4D3Uzgzm0BiozwipAiEAnHP2IzWgumFUVcSaC7in34Tjiha/rfbOOIoEbv67KDc=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415276,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf+Os2CRA9TVsSAnZWagAAM/MP/0r/GH3bsPk3ETnbMMN6\nZjjDHrbPkHKpqiS6j6BlHWxJykGyOoYRogchorC68tEvYw7D7356OeqLlz4G\nBZKu/nf+VaRZRfY3ssFt5jvgbVPOSjcN9RcWwIwXx2dNHRI7x4q8Z4Kr2lgc\ngeOyTKveQOtd4NwQgjVUBVtOgAJAYTATeln9ntvlhs/AAuTwtqsstzFTZ952\nZqoTKmWxkOWt0gdr6/k24oTWWUJil7kX9S+Sidb96l/c+acbyRpTiSApbDHc\nCAebo+185+6DiHm7PUreZDcH79PzeIZageeChA5d19QYJ+hpt8X6i85d/zAy\n9KoX0E6hfVRaYD9KMMzuGs4TnLO8+RA2Q3ciYGh2UADowLIpfjIsQ0STb4Gd\n9QOX+UYEy2gJJOPgfMXdpGAhm4rEIHsR2R1aBUprfXFaFJYXAGhZMaMX4Q+P\nSscwr3+m9nybEDdDJ3CwMeRRx68B9WyoJvOwHj4KzLTp88rzTxSY3zcusD+I\nk80Na2WoqyGvgM0SI/6YycO0P/jmGv7IOiaEN0/579EeBlO3i9EyVILz9UyY\nYRJDdw8KTzTzlbK0csdag4PsTWDPDJRC94fqLqlDn2x30nQs5pAqN8lrleEA\nBtHzw4EdoB5t8dwOaMaadEQvfwGijaWJeYxwvc7GGk7cTtdKZYCv2qNQ9+El\nBF07\r\n=WcQo\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"5b57277808ccd81561b989bcfc37e0e021ca4579","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.0-canary.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.0-canary.0_1610148661682_0.5389057204389125","host":"s3://npm-registry-packages"}},"0.29.0":{"name":"@blitzjs/file-pipeline","version":"0.29.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"8a89cb836b59ba19ac9bca3247c731b7cd1409e4","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.0.tgz","fileCount":28,"integrity":"sha512-gq+JlR4Itv2dlLVlRpjS4YgMOMFJQ1vFzResY6GrW4ieTnNSSgnZpXZgaNQ3tO5tl44OrC7NY5xqGYdrwkOjmA==","signatures":[{"sig":"MEUCIH5mWgbsax0cRHKKhmvTcgQPUJrlyNs6MfHHFOV88wZ3AiEA8H30fAYH535/t8Xjhu6iFmqWbJOJHGBK7Ac3523EVlQ=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415258,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf/iN/CRA9TVsSAnZWagAA9Y0P/09HJn5RNP3I8FOPz3PB\nG3AOHB3ylGWJAJgwsmHIgOo91psCH7Q1LwMG2lnBgR3Th+5Y9QYldbYIVZD8\nsrxOUFipzpnPT+h8Ym4eyB07ITA0VMIXSlHT+reIznZwC0/LUyTYXJATn8Mv\nZ9r7ArZRfJc4J09CFULssvRYmnKI6iznwX3fmHH/ErMy3+ESASy6Hft9L87S\nfOBJpk9OiqAsr2q9KXTjr7z5tpR0IRHv/KSd9xft9u+lAGOO9pvU+rgRlgJ3\nYQdGEwkrkRoU99mUVxJxZRdyycfc+EHeA9jyFf5Tt4XYageCMNMfPAW5HtL4\nbbMv3u0Rbl8uuh46lKyegBgIUF20H7dleLPcW1elrklA6n98YZU89/xih38p\nS/YdNq2EusVvbTFrh6Hrsg/AHntUZVfaIcL2EwPHpJC9Oa2oy7xPxxooIF8G\nnHd1B1dSfCE/5D82/piNkwYMqvaz0ie9aAsYgo57uqk2kBP9/TF2oj+//+G4\n+MGlZLRVc678sJVzAZuRE5vVSu9/4CN1sEJ/7EXeEMPsxRz2P8xsuzNl3Hq4\n6lyWOIpW8nUX8gaAv7R8apAZx5JFexpmLkKNoedamNJTQrBd/O31t6RXmb0/\nqNQpVltUYzaG+d4ZKFACv4rwDHjW7u16oEQZHYzUEzEnAhmrLktwGlUcLck8\nUMLc\r\n=DMy5\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"a2a5564a9628b6c875b9b69e6adcd19e67d06eab","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.0_1610490751119_0.7029877897604027","host":"s3://npm-registry-packages"}},"0.29.1-danger.406c2ae7.0":{"name":"@blitzjs/file-pipeline","version":"0.29.1-danger.406c2ae7.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.1-danger.406c2ae7.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"d064ba8951da69d02e33df86a284f791c5498212","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.1-danger.406c2ae7.0.tgz","fileCount":28,"integrity":"sha512-knf2ly1tLa4Qbef23TPjh+UJtmdbx7tqzXYfcnKBeWiGaN8smWQZrS5kYYBV7qtRpsRWWMhYKLkfupmXah4YDw==","signatures":[{"sig":"MEYCIQC+SBiVPMzpj1TBqmI8Ib31GK+hLn3mrFI/7Xc5hNYkkgIhAPFiT/0FG2XSMV3Bibr4NTFLv7AACo1A81dgWDfLWVlp","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415312,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf/jSMCRA9TVsSAnZWagAArVEP/0sk1cZ66pgW7gkxkGY6\nl/oJX2wXpxSM2GICqd5Xoi5TIxx9GiZbzvjR6BLMReMpF24myEm5nguwpIWw\npqqGJy2Q43ReTZhczfZKYfx+HPKaasuV8k3/Cb5RxEVoUJU6wbKSGP1H5BYR\n790d8fhgtCxZbySEznbp/NG3rB3agpQjOgx4ZTyZ+uXS+KirZYfud9WNnAkp\nV2h+2/e+AOTkhHHCBX4dziK0Igi7FjY1cYTtriAmvosWCNG4Pdk+PN7t1Kc5\nC5yw4fsbYux3U/YtB9jtJVEVF8Ez5Bn58ZsKGapdy/cM7L0F367hnDRW/rwx\neEhk++zZWE7W3SRYbO3aphg+3/7YMLsDuwfsYM/Snuwxi0iJiYlpHit0Tu55\n77mKD58Anw1HlR5yGj2XldJZ8J9o3j38AbPkEx0CQRvoIhv/EEuJfILeuhmy\nX94z9uym7MDpoSSp201EUDMFpSZEYLGvs37+8UcIJuRauoKiN4pm5gUMtq6t\n7mq3h9FFa8F/7n9DTHFkv9zrS+UIMifVp1dy1qX7dCFIzNzazU9vvp0r1xwa\nlxADUFWm5vf1ZlYAiNCVros0JPTf8YZwzGou5xf8c7pRrTxtkePRBgp/UvLo\nRmzG959K2/vyypiomvR0d1vajtQhSMMkhbuVytr1cpCLacPfD61ZQgWgjvf0\ns2qO\r\n=QqG4\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"406c2ae746d734fd03c0e0d78558c8e80db1f664","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.1-danger.406c2ae7.0+406c2ae7","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.1-danger.406c2ae7.0_1610495116014_0.09035107683959542","host":"s3://npm-registry-packages"}},"0.29.1-danger.796e93d9.1":{"name":"@blitzjs/file-pipeline","version":"0.29.1-danger.796e93d9.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.1-danger.796e93d9.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"06d885863e6d0fa44e855da97b2058e6fc5f4240","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.1-danger.796e93d9.1.tgz","fileCount":28,"integrity":"sha512-0F27AwpNhIkr8gc+NGWEP8WgMT3PYbiOGU19qGSBaGQ2TT7pfp2u6juPfAfsQBzjrNR+NtFsP7zVU5DzdJ6y1A==","signatures":[{"sig":"MEQCIA4v2xEHcZzhqQ4cb94rCkc7ko9GrjmGKL1WSWBj0JmKAiBcoN1XmFmcIl5wC0/mbTIm5hlC0Nhuj7zUqkA7usjJXg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415312,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf/jgCCRA9TVsSAnZWagAAx0kQAIE4PrxFhVWtrFi88l4T\nTclz3onfulUdjgWn7NoR3l49dFOURdpJR1Tqw2HVFCkhCZ3QdtZriuAGQqnS\nv88qiSn5qoVQ0hP+6/ZqiSxslwJevLPmAZs76MCg+acdMVheRtTfrgWwsYE0\n6/3nW/3siTLolxw+sq3EJxNLjZQATrRQvROtLLA4epE/4wBuAzj8ac8GZnCI\nL22nIPduIEqs/ua0fRexv5dSzl9JCxAJmqnOG38GxqVjJSCDbAV/tf8TIGtT\n1VsvJaN5vXrodUWZf1a+jeV5sH+LCjZQNWw6yD5fxrR7Vbf4wy1kdAO0Me7H\n2mj7os5N+8W7uHoRcNusmK+/pys35SZAQZCdUA3wcHKzRqefrKk2f57gnSOC\nTx88RRvi9F/CjXKu71++iaaiWmGo2jgGslNuSV560FmwJWlJSNyJRENYotSb\nlZT+kgTwBd++HE+6yMG/eX354G5gBI1MBcGPhqRmg1Ba1ezhpWivqMyOegzP\nrnNIxArwGi3Yscz9QiZaPqG1+SyrwteZZ9y/8kOF5dV7SLNWIGmvfi7B217Y\nnk/Wj15P0UzVhpo8i3+FeGB6SUjcZTC7nrFB++yqN5Ts6nWtiZPZfZRYasA0\ncT/ZPFhAnTm1VwwpxHvLQ7rkL/B7rESIksQc7YTFTlZpfhxr9vYItLsiWfDu\nEKc0\r\n=phRs\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"796e93d95f3d895a5dbdfb2125645ec1676d6c6b","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.1-danger.796e93d9.1+796e93d9","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.1-danger.796e93d9.1_1610496002029_0.8013555939096471","host":"s3://npm-registry-packages"}},"0.29.1-danger.f783246f.2":{"name":"@blitzjs/file-pipeline","version":"0.29.1-danger.f783246f.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.1-danger.f783246f.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"7622ae88cfd37dd51bdf2bf833339ca97b5d0e17","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.1-danger.f783246f.2.tgz","fileCount":28,"integrity":"sha512-bmsF3SJhsEBAvBbqQF0ySOIRcqiy5QuNOnvjt+TXYdf0NCntdlS6P2A9jwD05a5+JjCG70Wt7pIHSLOOuNW9Qw==","signatures":[{"sig":"MEUCIENeis4VkCMCzqYc7mMmeO8m9V0/Q4zT601BT7BV919WAiEAguOMhtxy47Y6T6Cfd57sH3ILtd7Go04jrAv3X85+CAM=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415312,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf/jj0CRA9TVsSAnZWagAAgsMP/R98uWF8ub6AQWdK+JuQ\nb1wMcbsPnVA9XwOIsqTMsBLtmJRisZ9o2Gflt0NxRlm/RFKTkBwPoRLPKVqk\nXz6QsII9saxn6A+9e6mUySJ5Aynzz4q6+cYhK+iMYJrd7Xa6WKO2EJGMW8jj\n6p7mzVnOHrN11ppfRfiwkB8EfKqxjildenc0t5FTzXS9hxCO+umFv6zW7M03\nOGIhOWaOgvQ1NKDyUs1UzOBLJPgYq/bTGyoBQ5N5k1HBO0P7sB8P2qRsA1ON\nKiGNFYtgVsbFxt/fBNzy7t6FlHcfTb19bBnTANdRQvkxYIGTeVDOkDqEHW3z\nbAyq5FDoBg/Ms3AF3wzc+JVNaeRYSiU1V2RXXa/IMiZNRGc3CGt3eILuUP4k\nMCaQBshXs7OLAqAbhaW7Phr9cLvPqkmoODK54ApqacadTHnWx75lGhGj489y\nqL48R3EOTeQHMhbVvXv9jJh8AZ+Hr7b2O/27AMPQ4BN6Lbfv9zm87GEKfGCy\n/snsqi0bCznwCgzlJlr+F5d6igbhuPcq2XezmKiG6QAKMfy1x6F3hkYClfuX\nEUmWLHl3ogsdti9KubtlAP7jqtJq7kDt1i0Rl7qegFPkkRTeqDs2IAxZyCH5\ngGE4L5w0wHuMwTBjgRnsk6La0i9nM9JVuo4PkOYMVusa5/P1FQqqqNbJOdJ4\nAHtH\r\n=glSp\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"f783246f38543f9c1847bebe146d7e9343409185","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.1-danger.f783246f.2+f783246f","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.1-danger.f783246f.2_1610496243895_0.9990771867338093","host":"s3://npm-registry-packages"}},"0.29.1":{"name":"@blitzjs/file-pipeline","version":"0.29.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"d769fab45e70a7d921c40eae8ff6f7b037dfc06c","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.1.tgz","fileCount":28,"integrity":"sha512-bcxyqoKou0j4kd9W++BcObB5nMuH0P3CDjeuXli+5oibQtJTObdcXcxVZ5HlPTuayVRbli8QfaFISbG1OCYlWQ==","signatures":[{"sig":"MEUCIQDIQD/9DRFRD09foINIRYgpaXvfLqZenvbfyncWhGmTXAIgbmpq5V3JQ3Qjk2V9FLQvvhEUeH74uUJ9fKfji31z/vE=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415258,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf/j7wCRA9TVsSAnZWagAALswP+gM/95PKBH+rQfLMs6Oj\nnDL04pdhCpuQ0+fLQwkQHyCEyxQlKv8j90ODXTQJgHJTkgzjRdxzmZaEMTLn\nrBWBhwCL5O/MoOgQXandMMxDrr7US0mePKi+8T4Gd/D2lVw359gr527YelBV\nNArsk7xN/qvEhkItvxJA1i81FUooft9ChFqJb7HIXMJEdOlWTzHwG1ipyf1U\nwSHmHpkH3VNB31NzV8jee99FtinrsdYXdTbwly8NYRyQfdZCkEDe+Y92rkqh\nbSGhsVQ34mlbpswP2VpAY+1VR8d13cCuIbEmn6/kCC7zA0yR9DJmslgdck3L\n4l032hUfQvA7eAkLZ59G1dEfEmqiX54ntlwm6R+6MaWDlcY3i5hMjwdBEY5B\nNcEsUE86EjtY7vemgXSy8C5gV6i/lF1joRgtTYkwjfOPN6mv3NVIMdZCPt9B\n7+dVghhxHSn+5viUeJXaBKfCQeuh6pB+hCXn2cLm59WDmMyUPJng96X8BGuR\nB5GRzDdH0PEKDsrzrzdZani9dZPyo02UwvUbfWoWI1aPQxH1fGj4vuo0Ukjc\n4OS2aex2owVv0qQxA6Gdw4JFbs+qB/MivEsHDzvVwHBbLsAvsGFImduC2UUe\n6tAj1lccYVAPqpGLrrdLt8lm+++mvGpeJqLazhxhQzpq3vft7YeOXU7xW5hm\n6Su+\r\n=CayD\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"103609cf0b5f517d2e75869c2148fc2c50daa243","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.1_1610497776342_0.8944299265156073","host":"s3://npm-registry-packages"}},"0.29.2-danger.cfee2e9e.1":{"name":"@blitzjs/file-pipeline","version":"0.29.2-danger.cfee2e9e.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.2-danger.cfee2e9e.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"5a34b361a4496b7ca3232b2749aa4f71e36ab5ec","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.2-danger.cfee2e9e.1.tgz","fileCount":28,"integrity":"sha512-ouuzC9qnh5H6MIuktUx/9e9UbmnfAlzXikXSzqHmryiO97GembyXvv+MRTa1TecB+TZTWGMJtjKqZFAwqENpug==","signatures":[{"sig":"MEYCIQCC0SZb4LP/t93cIit/+rG7JwajZwEmBcIGLOwOZY0dLQIhAN1dxdbtFH5fkdh4NmqETeJuKyPDYuT776gRyL8PIKSn","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415312,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf/3H7CRA9TVsSAnZWagAAG/cP/1Pqs2lLfLpsxr+SROxV\nomAS99tAWhCwvrpbuxDjcV6xq/vBexwGqSe2OTvcNgDj9C24BItjxeQLb+kO\nPQ6ck+qSgmYG6DD8JtjPKxCfO7rDP/6Cc3xznNDrMo9ukwMMaF/UWFPhXeWi\nTwIz6pl6SFau0ofzXMjtjk1dbtIFWJooI1JSGNfwtu6qR/eHC2+V2vATziny\nfXcmHR48qh/pFGuLafIExVJ7YLsPb973YSkFvis1MsFJcITcIcTNuaBHl0a+\nlQuw5RB3I4hTb0vIH2jaPTimbMSRFmMjkbT1nYGXfgtoOD9T1a8m0FL1gCgX\nBq8a87nW9EVP3JmTNsSXzysM6mDz9l/J0S6zNsrim9czcq2PkYhfZH3JY4iH\nYd7TkeBZ4l8ur6ZEQ1mIUeqkjPRr7GvQoHCAQ5CvCyZPi4zsjPr3BnSYhITM\nx6zfC1XhqYELA1bsQfnL0Z/HwATPWEuHON9ugbAel4fdEoqa5yhRx6CF0XfM\nDsQQE7Iv0Txk4MbQq+I9YLZhQm0+ZzZOrI1Cfaw7dYYpvFGD6J+7QH5LnkKR\nQ9Q/dBb7tdqKUfO2p3AzGr7ccEByfZMBF7TaSi4OCMdQgAhzsXVrgT9602YL\nksuWGrd6CDBmOCpOD5IFXDViR6wVc8ufB2uB+i+qNTqHPpDnEqKnFe/16lpB\n4WVk\r\n=SmBY\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"cfee2e9e8f712791a4f523bfb5cf5801626d47e9","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.2-danger.cfee2e9e.1+cfee2e9e","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.2-danger.cfee2e9e.1_1610576378952_0.4115999991560242","host":"s3://npm-registry-packages"}},"0.29.2-danger.9c6283fd.2":{"name":"@blitzjs/file-pipeline","version":"0.29.2-danger.9c6283fd.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.2-danger.9c6283fd.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"8f61b4f861d15fe5d0504f5dd321b4db85efd75f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.2-danger.9c6283fd.2.tgz","fileCount":28,"integrity":"sha512-aKBAaCnqW4DCE+IdlrEpBzFP9lKKj7NcVT6xcD2W8PXfoj/C6LIrYsoU0YB7DjbCkqCtzERTlT45Ws29NXzT+w==","signatures":[{"sig":"MEUCIGKYVhjfgaO0IUgN4Rf1gEUuKNMvwoYrvkoCkWSVL3f1AiEA85seHmvzKsBj0L4jbK9xNuurfUX1iENQIHqcowl2Jic=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415312,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf/3YPCRA9TVsSAnZWagAAy+YP/2tzAG0myzKkVLqDMjbd\nSKKB3ZyN4PAL4IhQn+aQ/zkMZdJKcHKNahcIzXGDvRdyWjvGep7wsSm76Kq0\nzYNtwtnxPvAxgkz0BYilgurY0O0hDC1XwZiGXliW+ZIo/xNEHOsi1CpGQq5v\nhb6hJzoOSm/vBbfypQOQ0oCc9222TDEV2FC7DoewqCi2/QCM3yjQM8YTl7H3\n9C7/ap/XTqkhMBPqZMGPM6H58Q8+cbOyyBIAOl0UKHle1ouJOIT/KMNYlaEU\nnkgmhloQmp1DlOGkTtarcwODYPjXGVhKmiy5HUL8XMJlj4goBDm5W6M2j8r1\nUi/EjJvg9lBVAsctjaUQxKzSecAeHgBDV9UlnZQGdNv8c6+pUrPG16J+/TTw\nPagATixvOWT5XO7lhr+3r9A+EMgCvGqPmO7Ik6Dcu5ahWy5ToMkFnysmOAkY\nIbVfE70+8RQoLzT3d1NSjRqm9b8Eti0sYfG5rT3yPXSC+xHQAECCTP004nfq\n5Qfw3qi2tvqmeMopZXT8oOdHnYggKT+I7EQVua0wVYBH6e2rq256z7zDnc6P\nm0cVS3J1vG1/52pREQLOLfBKpR6bL38Dppz9diEJt1PlisDPrA7gNGsg7xMf\nupkLfYqKpzVcc39KwuwuZGOefU3DQf2yTucd7i3PB7TrDHt44wYtR6fGkWl9\n0jjP\r\n=jJ+C\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"9c6283fda7c4e695e3994ca9d517b36679e61092","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.2-danger.9c6283fd.2+9c6283fd","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.2-danger.9c6283fd.2_1610577423392_0.7636902773102536","host":"s3://npm-registry-packages"}},"0.29.2-danger.c73eebfc.2":{"name":"@blitzjs/file-pipeline","version":"0.29.2-danger.c73eebfc.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.2-danger.c73eebfc.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"8277e8af36c7f936fd754d75871359948d018d3f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.2-danger.c73eebfc.2.tgz","fileCount":28,"integrity":"sha512-sGknIOelYfqo4tIZl8l8KJMjJoGHEWj0frY0YsQdKOk7yls3eUGZbyPmwRBDXNKwlLgnpl6TB1KYMq8lPbVkaw==","signatures":[{"sig":"MEQCIDvbvapVG7L1BdEBPJuqnNXYBhCVCL+bl0UXvyFCznmdAiBo694+qLfcMQ7VRUr+/6Ou5362JCVB2DNh79QCGB+RrQ==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415312,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf/3dnCRA9TVsSAnZWagAAyfQP/1VASJEcyfBqZuv4qfIo\nltVTYanlXKU0nqwxqb9ywPNVAAoyhEqz9w6MJMumuoT1f671t/AoUV9S2QFP\n5hmmG42I6zRI6/UXc6tw/lBQrIwKk8FqylGBB4t1PVZ7aF8zHSg/WGeynUs5\nTdsO5X9E3kRBeALGvNhtHMQBiM/5PEnUGBPZ/F04TRXMRGnJWhXBEHWNavgM\nvi20ppMNjVpgtEKyY+VklQTnBfCCW6mP1llqymzsWXmMw9oCZI2IWW4p1egO\nGDNFDeUVgQx2NR4X4wLjp5SjwQRE8jV2DglzXQnlQmmX1fba7FoONq0m/EQX\npkzFAq2z/TMlQPB/CRsXmYDv8ov3gEzVUS0bAWnW4OMejGVlK5PvBnBdmDYL\nkYT3Tda4RgFvpnQ71cnF7G8pX7PB0DTURHt49Dniq+E+HC8FIA7dOjkc+o8E\nQAIXZmgQwATeLlULxW0SAsWFSOepKH16o6CR7q8hItLEaWKuQcnApC2DsVDa\n5uLyDC71BXO0o4tXdoX9nHUoJNRWPB81RTDPEnMUffhJot+TrqGwAZKtatkH\natLrEtuC/cZTsqUCYLTYhbYHzEKZQa4N85zoV6SVAIvlEyKh7heNuOMNjztu\nQIuVSNlD2/XsB887gcwxrnmAXWhMchYasVDVgEoc7N1n20VQB07BiqvJcp8Y\n9ECB\r\n=D4Vv\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"c73eebfc930ef5281d327fde972f9cddb7057983","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.2-danger.c73eebfc.2+c73eebfc","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.2-danger.c73eebfc.2_1610577766520_0.5846520860226383","host":"s3://npm-registry-packages"}},"0.29.2":{"name":"@blitzjs/file-pipeline","version":"0.29.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"9b959ed50509ce92c93dd035ecb9c55d8a2ffc20","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.2.tgz","fileCount":28,"integrity":"sha512-gPSU7dybN5iCvMG6RYGRzVBsk0fpLJHj7ndHGjEUR2lpK4syfNE9/3r9gPsPkPNCx/jpXSWY6TCxgzxeYZRicg==","signatures":[{"sig":"MEUCIERvpWx6rgKAgYbb/q9CnWB1F5bjkKl4GOM2LHFzhPWWAiEAtt+jcy9LCMxWmuYe1Rkj/kQXJ+U3j+wlWKk+nJ9JcMY=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415258,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf/4MMCRA9TVsSAnZWagAAtGEQAJlIuU9ZEJlmeqYXHOmm\nV/x4AVICf1gOwfE1tdikV06PNAkDkE24UX0ral6YvIpDVySpJleFLkRM22A9\n5+DjqOpYdPVE9RvyoU+iVDvvTGs7HsbkVkYXltpHKo5FwuCDT9DT9Vyi0LZw\ngTRzV73R+hfZZ0DLhI7ITUjt+z0iSOPXo5Imb4/DVFLZS2RXFBBLtbHCHDGt\nDfQCYhcG3jtVjSKKIT9eSIv0wolP5ARbBNlQk05goDbrW286HfVHMWDjsOPu\nqR9ZguznCB5iEIQvAMEtxpFJYGpmyN7NSDJFqG4YPUoZCuj3dZ+f/3U3vxcq\nM42lk+LhGETY56PVyLyZDyJwXGw9SYMEUQvblyORQ/ScuVj0ZVba9VonNFtZ\nWknXEt4d/jHAZsPwq3q81TNxO5OPuw3IznEGgSKQCDm8PqeJT4HUnCH36HNT\naGmNODRPZKj/xRwYaMzFBJgZp+oN+vXXDHzIjeHgjx8S8Ins2HXAnX+q8DuN\ntRtbzY5hMF6JBWmnJaG1DGAyJXv1+twVaPOinWbxkmPxunvvlSg54A+p1AQI\nXm7Iz4sRVTBsjvIWZKbFgkf9Ud9li+O3KzKu1bu99rYF84xxx3h0lIQFiL84\nkY36jJoQSR6hY3R4hS/JZMLrXbRnOMleyj2rt6Kj6oPsQx3qdWG2pt5eYqRH\nw8gF\r\n=HBTm\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"4b293abc410cd3ced1d505e08a86df6c6b94cf75","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.2_1610580748358_0.32812115823117494","host":"s3://npm-registry-packages"}},"0.29.3-danger.28bf8b38.5":{"name":"@blitzjs/file-pipeline","version":"0.29.3-danger.28bf8b38.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.3-danger.28bf8b38.5","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"ac34bf25efe5b6c3347baf5c566339c6f23079e7","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.3-danger.28bf8b38.5.tgz","fileCount":28,"integrity":"sha512-8vRRCS4NDc1RhWAugdxZo2yxsDmQHMjtX5bLhlysKIYq19oT/IKFDwthzxaXfFv8HylajiXbV0RyhSb+msQ7+w==","signatures":[{"sig":"MEQCID4WzBF5ZpxrOR9DvZAsJyds3klkALUG6ynDU2rKYsTnAiBKYxuQJDWCKUpGvuFibKSHMTy4jckJd+a5Y3C0X5Gxrg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415312,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgAhUTCRA9TVsSAnZWagAAhUUP/3fqdGwD1//m1DKQqjus\nVu5sOxcbSCRbXJTx7JhkiQzUB3L7nQebYbzmwcpS8D4Mck5nIqsz5w0vL5wP\naIq600SeuTK439OzIYBhsHdkj07oJZJbyih4+PjtWHUkT7amm+xJSodppaTv\nOOT/UVZJnWYpT5fF3IbpAdC5ffcPXDJmVOGlHPSqvhJSlmVh1ekCviRkJAF/\n7+mpEbZtL5ohHvWyndaU1xOu3w3GS1gbb2IEbEuraG4fUeSNMbpEuUYcPe1j\nTCYT3B4luPis+QO9++vRyri2DGnacvkkIYP+qQnT8vGAljyQms1m/W2bXRQS\njkmfKbYA/95AE7n5pmcm7eLa0hpNt5EpjjoPkKtq8fMMj16tHhmntbkPdtao\nkU95tmNhcKhnlWBmMgOic3NPl8/ELGd9T4+kXiMg1BhnXwKrS7oNS35MjhK0\nKBDsR8BS8j7q8wFCZnLmqdcG7D9+wLqqyVWPEM35RClgNsWAxZC9F85cdpI9\nHXJcBzoOgxNn9lfi2IrvcvEP+rnNV91r1WgZ6kqccjsH5TzZMPP7D8jtH18c\nWmbgUYK7uMGf6VKogNL6K8V10m11WUPJFPhnpCyq/seLwKJMJvLou9aMxFFA\nfSinqJByUG/Fc+vY7QIuQSRDy+d0pdPF32ruV7gFvmL7tny3uhwvqz0THuLX\nK8eU\r\n=E62J\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"28bf8b38d2bb83fd5d7bd34b5ced619572be51a6","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.3-danger.28bf8b38.5+28bf8b38","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.3-danger.28bf8b38.5_1610749203000_0.9905640902423252","host":"s3://npm-registry-packages"}},"0.29.3-danger.d7fef62e.6":{"name":"@blitzjs/file-pipeline","version":"0.29.3-danger.d7fef62e.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.3-danger.d7fef62e.6","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"ba94ffc0e17688cba3ace86dd25394dfc70a93ef","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.3-danger.d7fef62e.6.tgz","fileCount":28,"integrity":"sha512-VAEaan5rlugWCM6nbw3WfytzluPjyd9fVZlWzjhLc8+rRHULeRGSkrPJ0oUwjxBdZqcvwHzMNMnHQh0R6d9wCA==","signatures":[{"sig":"MEUCIQCtmXhDR+wCHBT/9JJHf32upop9uJyREoOnHPbt9g0naQIgJX+JbZ37sM5XG7xLThRk9m3sMLGUS43GeL5CCNV6pbk=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415312,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgAhYFCRA9TVsSAnZWagAA2l0P+gJqK2rpTpH9Z/HPFNZK\nuQxNh5KxjkMEqRHiQ4hwATBZJ+1gf0AlKuez/rnM48HGKHvbCmOaNoOq9/Tt\nPq4XP5L16YurZYVnsYablDTBp8FITLJifhAEnEN8T4V4ivG28jA4k3ibjYH2\nWFL8IujUorURbNYaYB89/vvbKE9tD6/IrpvSw0ii4ckh9oiXFNRukrRM8/4n\nnTA4sa8tUuQvFJzHt+cLmG6L8ExmpmITSgAFTIeK6ZOu5vQ4BsfsPDrS+/E/\nT7J5QfCN6Ma96OZUw/iuQIp+O4ACbjd4moI+nPU0DShhsskIBMISMRF6gdWK\n4SaODe/qYYZ38MlFNNqTMwIcxeJVKVXsRZwSyCf2mIAqlNu/4bBnkSfLV1tx\nI3YjGwNlheZ/+mgOncD+MQVZJbfz7d+ZJ+gft6Jdd+8elsi6yjGvulBRhfEV\ngJgmebe4bi7adZRS0clggRpvKRRWQjC458lMHqGTdGP2NPdanmODXp7D4dq6\nPjZqqG4ex1mRsv6r+vPdxvJ2eVl494MVqDW4WFssF40PHX+PUYAb49UZCh/M\nnb/WTNSmOc1VLwFzB8v0np0SkB/mi6kjh44Y3/lBQS1qplRziJFv7jRvDu0m\n3al7gNbRz9Ev2s4E4bQEfG0aJI5FaxHc8ECXJoFWn6BQQLQh74OTffJQa29X\nOCjQ\r\n=lOu8\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"d7fef62ea29e8326ee7e597f5615a1602a8a6e92","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.3-danger.d7fef62e.6+d7fef62e","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.3-danger.d7fef62e.6_1610749444976_0.43184160747117173","host":"s3://npm-registry-packages"}},"0.29.3-danger.2b28990a.7":{"name":"@blitzjs/file-pipeline","version":"0.29.3-danger.2b28990a.7","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.3-danger.2b28990a.7","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6a7168d2af24c3d904a029646b309da7e36cb414","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.3-danger.2b28990a.7.tgz","fileCount":28,"integrity":"sha512-sUElsTtCIb+a57VsYw0EOlWAMxkD6kX6MoWVqOm63yex7xV1wkVgfLo3Glr5leHC5lGwxGeHlltmUXj1BlJC2Q==","signatures":[{"sig":"MEYCIQD9/+hHaA2Df0e4vOPlanx/zPhlGyMLvL2CRcbCyz8hZwIhALSubx+CKXUZYLgY8uRUcnbhgc2RVuuxqt7je5hYAVyF","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415312,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgAh0MCRA9TVsSAnZWagAAhqUP/j9wz1DNr2UR/V8ThoFW\not75hsD0y3EuZv6C46hoG7jbU3QePjESQC5wUIyXS4id9mS3lSxTuAHE+gKx\nx7frlyA0fysmmFjDFtTxVghD8AZ1/GfD/5tMnHg29wefdtm9BS7al7Zb/miC\nEQEXETHQ0dKFMouMCjGHiEGiPk/GFNV6mXJYtWmfv+ydtAu/jDs9WgXoWDbj\n+eWrfgghev9GU8gOp43e7DOZOTILaEnspthPSWRIJ4Uxd7UWQaIzrpQtovGK\nu1SfUad09jhqy9dgQj+cCzzFAbJLkAD5IDwLlXDKbaHTQclpOOoCUacptXlm\nL1oKVEgjwRvcq5Am86Rxf6p65PSY6A7Ww5O0GHS9bscdncxi5mv8D+DrXNpT\nUIZXtnU5PA8ESWtAJrF0k1k77ha/0QQGWRB5BOBCi3wgjCLvSa6vab4Ugz9D\nioVcWzIkPaBYNQKbzE7LgAPX2MksVxzzgzdpwxXHkUA0NxyQh2LktFKvZ4La\nK54WYUluUuwOQoOBotHXtUklJBy6MpvBhd8mMETSfWqRZti1+JV8uO09y9EK\nBIO3cH+iLxX6zS0wykHHW6hwOMZbSaq47bwiXzXye0NhZkb/oauKTSpDCybD\nxgAV+L+Xjvw2p0omlAyjMIPm7GyWD2vuK9ABor6Y5kLVUINk/iPkljc6ZZSq\nwdt1\r\n=SDcl\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"2b28990ad207c08331352eea89df6c3ab03bdfbc","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.3-danger.2b28990a.7+2b28990a","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.3-danger.2b28990a.7_1610751243725_0.11401752748859972","host":"s3://npm-registry-packages"}},"0.29.3-danger.5b44344b.8":{"name":"@blitzjs/file-pipeline","version":"0.29.3-danger.5b44344b.8","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.3-danger.5b44344b.8","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"567c1e28c6c7eeaa140ee90ab1f070434040b512","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.3-danger.5b44344b.8.tgz","fileCount":28,"integrity":"sha512-LVPR7B7lmn7QsUOnlkjdNB4Dml2tdM2Q0zbYPb0yEt6BaTSHFCqYG+bXmxSWcQQmY12rX738+pmKAlHfX34NlA==","signatures":[{"sig":"MEUCIQDRCUJnrxaMi2MU16FhNI+AnvLZ7/pGQr+Fh5jgNZ29NQIgfjG8kEVseSTdfovZL5nzwnxrFQwVYySj+jRlI4NIXs0=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415312,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgAh8NCRA9TVsSAnZWagAA3pIP/Am01d4S+7RoJjaslz/2\nbKMLDWXQIkEFXi0xA/eMiSxnqAnsjeE/MPRmB90u/Ygmh8KOLonXVLEP4ilC\nUt4OeljoRCPCVjHJ8FcC1dindrYic3xigUh834PM4CGaVXYLQs1iR6LRSnRN\nUvzqIBDyQ+WLygNEAkw/+ave11w30TBdKq0GTUlQtrAyLOd60J/EHpkZOa0Y\nEKQmj4PacM5C3jDZe5b9hT4XLfhDVhQY2sHYQ4wtR0xTJRt3REDZBs6dmo9/\nrJy2sdXUETtZIRRI0dhVDxW3ebiZIWKjDuVXc3U++YqnDCTCrMK77nUbE6oI\nVkI13pOg/zQV7ovqLZMYxJRFARhx5qfe0Qc4FBInFXJzxfp5+L6oWIka6P9G\nYUA+Nk3IDF9muJS7AlnvKY1MP1QCBwUpmRlTE/jpxIxQUujeCH6Sw9+bhoKz\neqbPiYzy8JkV3aEmB7ROwazfkIg9fCQ5wD2pigJ40hIS0X13pfLzE4FEHwGu\nZnI8U3KLZ1jrdHMV7i6O0Hq3FR1iSreQ+VnwvAiU/ukkBXFWpw7c3RjwwCa5\nxyp5mzFiAQ/8QLb3FM8AOE3KCWCuHlzOFIYsCqumzeifjZWAgZjiw0gYqeRi\npLt3kXndRUwj+sLMLcZNZmiypWjz1N2CZ2UGzdSwtQ8tzOOTYQL53TlvLIbx\n6Cf7\r\n=+OqI\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"5b44344b50103a780ab79fafa94e5dae8c516900","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.3-danger.5b44344b.8+5b44344b","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.3-danger.5b44344b.8_1610751756607_0.26556475396343826","host":"s3://npm-registry-packages"}},"0.29.3-danger.2c0a6847.17":{"name":"@blitzjs/file-pipeline","version":"0.29.3-danger.2c0a6847.17","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.3-danger.2c0a6847.17","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"5d40c5a4fb9ecc066522e12cb9b51f30cedaf89f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.3-danger.2c0a6847.17.tgz","fileCount":28,"integrity":"sha512-Jj6z+F2+CgDf9J1QhuZ3/OOlUDfvaHReU3mYMYFlmM21fPsiiCFthxSVBkq3W2kkbG3E10XwvqIwXvHFuEOCZQ==","signatures":[{"sig":"MEYCIQDpkZxOkFyQsqzXI42TaS7yg4XiB/L473xFRl7vZHYlfgIhAILfDzT9fjxOANIPYI0x9TRbA+hijK4IcChYTWAUXzNH","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415314,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgBwtYCRA9TVsSAnZWagAArWsP/RS/DSaA8xirGxN2chQg\nI5cdeiY1Qv2r7lrPyV1T0Kcdqzv231g8OpldpguWZu2YWpKVxVaSlC/ERxi8\ngrAWqzjLUsWdCybFKuUaxLiHTr2MsFyFpYl5u4ZocokUARuVQVaUGvKyHtHD\nVUs6Su9wJI8YoHQP0uMSesvyHRwTpSqMcuLJs/z3evgG1TdUUZZkI24qjHMA\n44inkCfa09uk6CKmdndIzMD/UM/zVjJ1MsVs08BJHldhuByewVNtTRzFjOoN\n/5b4XBYFBwgZmsE3cNy5xQLF5XooMdGNQR4F6Zf8356EkY0CgL4u9V1j4Ori\n+v1CV/Mfto0eC6Va0/e5ANe2ivij/TBkHnB2xiGEHzaEq07J47j4Le7iK4JA\nd/vdxjA47xOfuRmNCjnlJKUMXKie44/rRqKeQ5FjsQmUBmKYx8TQMq3u5q2l\nqjXgFz611+VMr/pwHY3sKVueTdJyzs1HTuymF642qGMht+JA8jUqsOLtCGT9\nA5aFCVWkff2New9Q0Ci5PwZAOY8qEOcsx0tWJiqz3F5Wo3zyqvfXdvEU29ON\nvrXCaOc3vw1bmHhJ6hpcZdqxDdf9YNoDBwxRfBar/7jx6KLFUVkwg8K8Ou+K\nUELk8I80+E/BnVsp36ykuS2fds5dwZLgBGJ4SZMigWB9M6j0hErQLjNDO/T7\nZ1Xg\r\n=D2Sp\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"2c0a68478ccb78112a263da7a6ebd62617bb670f","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.3-danger.2c0a6847.17+2c0a6847","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.3-danger.2c0a6847.17_1611074391855_0.48190706020956275","host":"s3://npm-registry-packages"}},"0.29.3":{"name":"@blitzjs/file-pipeline","version":"0.29.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.3","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"9110c3366bf3126d222286734b611a4e9af19607","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.3.tgz","fileCount":28,"integrity":"sha512-rkQoGWvX6EFMG9rBqNV3se6oF2fAXOiMEeh9boyWfEOqxxfneFmUDTHpeExKdOSfMV0qe5VLrQJecYiLwSdNMg==","signatures":[{"sig":"MEYCIQCHndEfjsqdYuNITRoLLg2Hg8vSkqMHsrRuaDo2HTaTzAIhANuRI9pRVK8kRaA/ZjpePBTFS/3WiNhtpZMatRTqH59Q","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415258,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgCK1ZCRA9TVsSAnZWagAAymYQAJj2ahCLURFdtlqsfAOE\n3YDW+w5nAMP+qNk46acy1+r4TKTjsfUvpe1u9ErHctkYIRcS8KMsyDnWT6O3\nh6H61yIQoJXR7NU/WUiSlee2rxi/j/aSNzn+QBA7YW7+6ADixSj1gHueh+Zs\nU/QNlQivuQ4Ot1s55hNps7HS0DrVpnvt+d93oxadkolCz3ineeZsl8DQmhu5\n6b0wGVbVXTn7sNFojoO6s+AZQ+PI3uPxWpUCiTPQIOApXmmEH1o8FX8tbppL\nL/Sdek3PijsMtb6qIz3JKT+ucvldHhF1pbSEO9Eu2TuI86TJTQAYWW32vdx/\ndbPEmGSYr29mx0LmlaPzCg/esBbhW7GazeV6iY60e4V8DfMBn9xTFMLpN//Y\n6pmUeil/yvuM0UORlkv09znK+5igj4NZLkJy2cQaIZsPOsBFFx9pXCHBNC2H\nwX2yviYRUeCg7wd8SMZi8hY0fiywBS5n083FAC9ctfuECMULw7Uk+A5nVYpd\nOw7+3lqc+8dCqDXaAyLVuMv1EAiX+6lt8coEnHttsSTihAhGFDMCCUVZAlmN\nULIU/Hf5Fu5L1aDq2XahJibT2ZqKBOY7B8Dak/JLHgj7Ftfsnbop7T6BPcpl\nAAdBWYRSSsYtRBTqL6y4ApxH2Xfur3JpO5l2aV2c8JAuTPrFM9Fyhh8ei0FJ\nutRg\r\n=dNUp\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"a0e6eeb7c5aa2e1048a2cfcea98483e46d56a269","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.3","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.3_1611181401337_0.02525160763530554","host":"s3://npm-registry-packages"}},"0.29.5":{"name":"@blitzjs/file-pipeline","version":"0.29.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.5","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"bf32d79393d7800078bb01b7dbc796824d7ece40","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.5.tgz","fileCount":28,"integrity":"sha512-wXcfAG6nMD6VzIwfNdu28qTopZoGm7bUzDpmyKiEx0VZI/2UKZwsx4YMwKyIzt8AY59ZH0foUMU+2M3ilRy88A==","signatures":[{"sig":"MEQCIHwtIdTtChQSwd7ze1kBFe706JjhkDP+I1mmPFtgqEcIAiBfSyfAVtIglenesyfk8BblsB8iOOtgjV4qxLRy9GCJJg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415258,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgEEjjCRA9TVsSAnZWagAASLoP/jFTJHAVyt4mNl1ulWEO\n+qpjzGNP0btFQ5H6Bt1aRB22u5OzrORrJ6qe1yJHJ/yZg3znZaJycyXRJ4sM\nXNUZ/BCQcWo32u7iwXOVld9K3QYhcsm3jMIut4DtRDhvNa8OugPRkKoz4mFx\nWrIV55tu4ExQ3XGKWGozyqkdkZWPJiYG3tiMKbPsGx49ukGvjeFthCo6YrRW\njRCn4TcjGzxYWWoIt1kSVmxJK6+0EvgK4GJip5kxoll8FdVLnpNnWMhb4ib3\nRVaUmAOoMbT287jWWxSdtmZGhfbvfRNx/yO7ls2ArdUVNyG4hxkVmsHs1k0f\nzqDc+yw4hpP5S6T+ke0wjeoi7VoWxAjG13kuEETiPAAQ8PMQMIptZifMgkep\npl8JC2LaF0Nr7F7Pk/YHKRy1U3UjHTGwFTOgdHaGwtaxZ8F4zhBibxrafRUl\n+b7SkgxNpWMkcC0A0qk2xKMM8XyZaCpAXAvzl7zRRaMPMOVdeOD9i6aNLEln\nLl81pWMjsOI28+y5CLaet6H2l5/1zYSgOaCkf+ePpSYCq00mANSkzaaeQ7us\nGrpAVk9lozxquybMTvGdh9hsPix6zhI98+r7gYsKtHs2nCl1o5yQKgnEWXVL\nR8wF9uFCTIbyfKG+q6WM1pGibFIiTwfphTU1pnhwZKyyNMLXTadYzL8p50xo\neYR2\r\n=fiRp\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"217c450363144d5200c8d018e8d070b5251f7ef6","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.5","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.5_1611679971521_0.9613727176015217","host":"s3://npm-registry-packages"}},"0.29.6":{"name":"@blitzjs/file-pipeline","version":"0.29.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.6","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"0da96d48ddf1dbe812fb0b2b86c5c868f0a5e640","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.6.tgz","fileCount":28,"integrity":"sha512-fbuUlAy2ugiPu5YJkKC78ZCR+/MyTL4RLdueTIY7kPdHgXlnesTRxp7mBtB1zT5zbZBRvP1fAtnPG6tThzpX+Q==","signatures":[{"sig":"MEUCIE5gve1k16fFvgW9UStMKoubDfO5No+AK2QCFfk83s0YAiEAmhPdA84sBdSQKRVX9F6DK5aCDfPeSNC6Bq9gXUGpVks=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415258,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgEYOzCRA9TVsSAnZWagAA1x4P/2N6xjLf1w3ju+HAxYyU\nnWylJ30TQgaGg41rcF69fcNj1i1/+GITTwH2g4iS6VG9cRH2AN2cSAAgMdhr\ntjTNvkgx9QjWSMdl+ZmoZdKpVdUS/v4EqD28FR1wuc9yDVEAmhXH6NSid1QB\nRvqRcQBudzEb9Ry8hLJqACjayIBWx8DzFpML6k1JJxcuXFwxvrKMENSEHwHa\n4CUWnZgod4Kk8I1C3IN8TSpV/3+M1br3Frybq1wRoT482cX7aDqVSOoj4veH\nZtQCOtE0zI5LYmMoQOTiBSksO1yV6sYQVuUKVZpHiwTB5ttakEO+PJpKU3+w\njpj6VzPDNFXrORpz79vIDxXdfrdmbP1fH3EITAwFZnagFcnX5qL4RcGL3JYl\nwG4j6X5nADlVqQEP6ObMktKBKewBOODHH3me/uxGfow1zUdBM1Vf5NpehaJT\nuBHwM5GuGsZnrUlIkWQWfQXLup6ejWzHqAS31N9m418MA8OJiKte0o+x8WQN\nNLqG4CXBATB07rzDaCvjPzXf4OVkPhaavCe/nzO89JRR/20re+tt4AALeIFJ\ndeu1+zuZn334vpdrJQeMKUK2GHXQZ44BVLSAzWIgL14JbKF3UWW80l1MXsww\nePmhZ142NOqnnoO1ucza9/WlhUV8EknZLnPc2qrLa+pBMZDWwquU/9Xlk/tD\nu9bC\r\n=45BI\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"7464d23a37b87d362aaa0858f8d412e8e18379f2","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.6","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.6_1611760562522_0.2836509727218519","host":"s3://npm-registry-packages"}},"0.29.7":{"name":"@blitzjs/file-pipeline","version":"0.29.7","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.29.7","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6699b6ed1be6095e74223ffe4e5719cc8344a700","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.29.7.tgz","fileCount":28,"integrity":"sha512-dVyEzQjMZw5xfWtduIPQLamD0QCH2ByU7O0cqJPe8alxv+PhDjdC6QfwUkHkYFFONz8N6s1D9GhiY5Nwrw4cKQ==","signatures":[{"sig":"MEUCIFudbqJakYYf+0cn75d407PnhafTWwg4tZ1zxNj28gE3AiEA4OBRywkBFNbci2I60pfwZYDSL1oiZr4AwaQreyo5nbg=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415258,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgEZmwCRA9TVsSAnZWagAAI7QP/2v9suEfE5PzuARj5Mtd\n/vz7w2axRMYCXEanzC+4uaEYaE/vRRBtBI++R3vAoEYtAkkYie+ORPGsEVWf\nIoXf3bZ2I2U5WKMgFvg+1F1mozpu8LJFT39DkP0y1X6hYy7xD3YsX/NUF1KF\nH/QSojPKglgQAZS49jJ+LbivDUoqKbkZBjRZrxlimW62Eh4r2X/8PlWn1bh9\noJiVQ9iGkbgUTfjapofVNthkq0EtxteW6jDPpxTmt6XTvbJxNhT3REfJU8DF\nVumdLhqP6nqJhkkxoxcptxRx18nMRMHKuxkKAeA44TDc/RDeQxb8SC87N3kd\nZVlWpLPahSwZINeU4Q2Xq9vElnw3Dc3Sm4sWdM8dec+8fd3KwvW6OfafjVo5\n6EcjHp61vOWykqgn8yRghkHaD+HP+BM6CwG8Cbn8/YcLnd0Db5auBbIZIVn0\nw8eCFqud3h9MZv4BFvv/SUzlap6zWZWlHxbaItqLYRkqRCtB5u/9k1aiZx53\nqlEeMK01Ecj44//Ic6bUabVXnPyJ1TR7B+CPb7200Fide1c/nEixdyPvbqNG\nzncmio1QSYLyBeNnZwgkK0Wn+n3Zz2wBTSCWqMduCL9doYSZjM+57IDGXhkt\nTtYCP6wAWai9oaNEovDZNmmawa4zbkKohpNu4L1YHVeORwUQwqLqWHGj4sSf\nlNlo\r\n=cYMk\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"e565b21fa099a560c1e6398745bf344032211c1c","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.4.3","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.29.7","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.29.7_1611766192223_0.130250925044622","host":"s3://npm-registry-packages"}},"0.30.0-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"4e85fa87a033ca65bbc9149070f5638e8ae9d3ef","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.0.tgz","fileCount":28,"integrity":"sha512-b90ybFQXVRl6+e3i6IUZ/Ov3SsghNK1zy42x8dAjfm4KzgUcnJkazQ2qfoRyJrgNSrzjh63Sau7HZlr1lO3HeA==","signatures":[{"sig":"MEYCIQC46StCJcy/1zjPzKYr6ZthhW36FM+snwRGrlgjtZkcLAIhAJT3FFfPcnh47XLekgQMo6Srd94W/Qz/WNF+ULDrUbmS","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415276,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgEaONCRA9TVsSAnZWagAA1kMP+QARqm5uUO9NS7ppkEqT\nI24fF4KISKEPIC3D7xopiLzEdNaeJQXpcKrfV0/kZTtdyaAL17UF9PXyz1Ls\noKJLDsMnPCvO2HqtZ+755IDWF54Bpt6OTiBnEwgJx+Ju00ZnMbKS229CyBED\nc5+jQruJhO6HwBA4VheKvnJ305bam62ei+1ZlWZr+eQ0Mp0Dswb28nTGx3ER\nlXEkiRdTDZ9BVVFmh+Ly5H4oDp/bsYAzRxIMeuMHoPPukV0hQ8yc/W/oyHi3\n2bj6W9+diFvW51STYy4B9ge9TSoZ5CyBSGQtvkB3OIswMz84/vX+6hv/oJiU\n7hM9/djJbNbabDAN/pqD2glDo/gL1Ef9vN7uboemJH9Qo6I2fLnQIcPxZq2B\n2Jq+2GTfVJLfjSDILk4cqdDU6I7LF/doCkpXK0/srlj/H/dJ5xS4e7K4zN/3\nIJ4LO+EYNeG1xYjDD1Pf/JgPuiGQLlPpPbJuFMMsvtS194z6bDHutctVYKsy\n93nr/p9zcPb385JsINYDNQ7aYCbw10l+81c1SDrLcV0P7epUbdBQ5+oILICS\nGzAPN+D5Llx947s0skVxF55GkBbs6/wie5OZJMkrKLQRs6+RGOOjTD/BNxX8\nI12S6wAomgasn5/tPh4jj2Nhnil/HcS3QT7IPnL7ivR3B9+j9KXFMTyLqdeE\nhc2t\r\n=imle\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"ebc6d478324a8a8aef41be553cfeeb018cf4ee97","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.0_1611768717002_0.5941133146911939","host":"s3://npm-registry-packages"}},"0.30.0-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"59a8505cb33eae4262ca9557f93983e3e06fb43c","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.1.tgz","fileCount":28,"integrity":"sha512-cTJnxJdHhJFw3TrnoQiagZ6K+hNhsvzLIA+xqUNi/NogAM4AO3xyTh1cpaveahfqigeb3TTtmV8U5JCAILj0eg==","signatures":[{"sig":"MEUCIBz13fdTt5j6+1EETK8TYFbh1T8PoGT+qZyN0T7PuMBoAiEA6j2zx/BOigWtfkPex+bwg9+M//QZOjWC9lvOlSnYx1I=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415276,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgEeWzCRA9TVsSAnZWagAAAKIQAIjCVKCseZX/xshxcbQS\nH1rIpvjNguFwEao0OjihrDTdKkeBh+aZ4q7WpcRnY3chNvBxGPKJrWJCawsz\nWffeAb/sAXE7y4xvzk+7d4+OucZmj9gfxLGLCRpMJR72irotFNbJmfjRCfgY\n405wXdYzt9rDpyUMA3YW0WkFBHceh5p1nEglYnWb7cQE8mmkrT1wepuydhYF\nLn4OGGoN3ApRdfsLtnaoBjoJFEZJdHCWuCTAMy40Bwg0cyVve4ECQKbHd+lv\nV8MTlpK+x/yy47PU3Q+3N1+RuFvXJkj5akPyMaA/e6QPK54U3iWaLWJuvlZp\nCRY9ccfewDywQasKBB2o4zhkWSBiesziwzIG826Ci7l98um0QJbmj+Ljn6Zl\nfYr0PI+0nZI8+/0Kd62eMeDkctf12tSRCmVPVLu6uyS63ZSJxb+AwAwzNjDN\nrkkj7n3/IaWKrQ3V/CFItqDcFm5wvYdBXQnxQYBq3rCVFqVWjLki/Ct8VdMy\nvfvUhTv/5l1q2K0a4j+/DIvAdb0JQeHW6LJMF6s0NRmWTHw4Ckzqg5D9lcoj\nfaFA4HzWpOTgvpGP5RqZo53A5g60BwKXoGJ88G8vyAhvr9LxQGI3sogkkWY2\nkdZthWg6xc6OivUzXl0DZ0S0NH8GyTe6RDNpoZrKn+B4mEvEx/bJCMJOHmc3\nqRxc\r\n=jBWu\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"fb8a13b04c7553159536e15ed0472b9755027015","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.1_1611785651283_0.43540255615588874","host":"s3://npm-registry-packages"}},"0.30.0-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"a4b7e451b5ae301add016a539f70c7e3ca00e45e","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.2.tgz","fileCount":28,"integrity":"sha512-vBUpVOZ2PqNZ0o75+WZfVP8yviZwG0Rv+1IGjoYs8X4cCEWj1trCoIHlUIHk+rArS0weWWUn7siTb72rZ18aXQ==","signatures":[{"sig":"MEUCIDpszHn1qh1ATGBue8qSkgNnDiRWcT6jko+Nn7bkkgzOAiEAi0KxUButVHEytCl81SBxLEYRQZzGKUxbhXsI18NAiTQ=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415276,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgEh+8CRA9TVsSAnZWagAA36YP+QCjAJfQuJ+kkwbI/ApH\nX6Fnk104ATMxrv9RAOgM2e3F9j7ZPCvqe9Be20O83Ll3Om6lhu2BNujqtePZ\ndsR3b4tSLMArl+jK0/BYieGcu2VOZ9WYipveILszSKAO5wygefajFyUK+3tu\nvA2l+ZQN04/9I2+FK7X6gbr3YyTzMRhluwL6l9t7Lb3ng5rFTv2/HC/tpK5s\nC6NRhO3oI547GnkoUslyGYFkWOPUiiPOSUs/nDNWQgslTMQBULzASavkTBQd\nTahponO8kFzqrhP7F4lV3Vijd1Ma4SN0FkxMyzAtUoY7VP3Fs6LmKL1rq7dz\nhjLiu/gsXhSvjf2AI0NoTERum8NLh+vN262DhsVJ8wz3mihg6H7Hg40m/vys\nrC75A11qTLYa0IwLHtfXeYkOWr3vs4n+zNcVy+eQJZ1SwSatDuyunGGaI1hK\nV++dGY/B11FOMF6COE8+s+zCFC2JlWmJye2KvzUfVe2MZRDziQCKJY4Qg9z3\nHO3y8kpb8CPmhk8vtstZ2GZYoULnDP/FvwEmdGjeljmpRq9LzuaeF0I8TbtM\nWzcBaM6myxA5lFjhQEyPKVu2pyMcebf74ulIIrqTcZQK8CyvYcLLRcHp78Lo\nBJRW4aa1GngY/MJUR/wzuhNCGzmzzakyA9ak0eTSjh7DZpUNDao9KHz14XtX\nOVkp\r\n=Sxgy\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"91886728cf7681fa4b46980bae7871393b88a5da","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.2_1611800508294_0.9005216073236502","host":"s3://npm-registry-packages"}},"0.30.0-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.3","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"24d99675100aa733891fb23205006842b8aa4915","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.3.tgz","fileCount":28,"integrity":"sha512-ObdB5zlkEdhkSivMcsyCxPS1T+xNytw/SnR/MEh8mp998WoqSYlr3lXhC6+RhfjKiLpizYHSJAhFC5eOXMejWQ==","signatures":[{"sig":"MEQCIDxHGM1nz08aznmSNXYfHRkfxDDmWvW6yOnFjjfz4kQoAiBqI3Ak+jCqbiuxQin/Yit+YYlBPGoDEXnwYgrFGHkDpQ==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415276,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgFFoeCRA9TVsSAnZWagAA95cP/3qoYUEG8b6ex+o6q7uf\nbPCT0t6mpPiWjDUespadwY212wVNdTSWVsfOvY2I99yMQo2D1XwsrDMEsObh\nmDN61273azgbhN7jKYZmEU+kD5ar1p5B1gOZLeERpLR2Pb4k+mNE6BhDhCdX\nmf5QaAikzkOsvNJFA74fm0si2TZfqw1Hu1SqYBb7oRqict/BxRdauIUpZa2I\nv8UJiTkDvYfTrg9YdMvT89CfUNSF1YYgHW51YwQGc0WkQ+i1xiPD4zrWSMo1\nSEkgwXN0QT/nk+7H/FqKqYcMTsRy+502PIUB4JUdOW9WH2epJvX/EIQO+Ikz\n1znfkpUKnV8QOLoyv/LiQQ+xZWMQaJM2AbMlnDBMjkwaAez5QI0GD3H/mVmm\nPSM2kI0iYOn/+PSGN5ZesSz6UtzzFnUN/fclvjxgvBRo+nc5Moe5KkSHAET7\nXJ5Bli6FH0L3Y0vhaMMQBcPuvAlzBgTEOF5gILqf39WGhskv+gvpMQD0HJuT\nQSshyYwOanfib0ZWNaDbCC0IQaptqXThvlpcwiw8z9dIzmEHbVYXWwTH5PTB\nyqwuHrAyN20w+PFDHfT97T+nkpjGR5txYitrMIM+AwkYPZwky+qnXQbS4T7z\nsdRGLqg3t90hM+bFTyc+1X0FjmHFpBlIsp4W3OH6Jn2kD2rHhkFel84ePPYz\nkqA6\r\n=LpjQ\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"a0615be2852790453d208eaf9bd9d4950a231044","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.3","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.3_1611946525177_0.6883738443258509","host":"s3://npm-registry-packages"}},"0.30.0-canary.4":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.4","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"b97f7ce1be3b008c07328ff80997a8dbb54fdec6","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.4.tgz","fileCount":28,"integrity":"sha512-7k0hDz/56WlJpHNw2lVdjOdZwvinhv6zjHad2xvN3FQQbdNz+Nb+blgOgWSTpmpAY+Wrj094Aw3qVwol9PHEbQ==","signatures":[{"sig":"MEUCIH7fxQwYFVjjC4rBvRWDpfAcc2RxQmGq9LXpx24wziQrAiEA4RiEHt4HaXGLEE1EmdDnMI8tilo5G222t86Vs3pVtrs=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415276,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgFMR2CRA9TVsSAnZWagAAcwQP/28Xr96nstMEzN0Aoq77\n2bLANo4Ca8ST7xYDkduCVXaRdigRKlU9Kxm5vNKIsIKuSbOAtBk4aegkMyeG\n45oUvjUvf2HsyyVrYzztRAagMLRv/V4Qir3OIWXRaWL/VwDFT2WcmiQ8zVvc\nLyJve+w9UbOfz6BOPLlkjlZdroZFwRfQnEfnHL5U7HsFHRoSJF7VaQWswYp5\nDjI4VzRv/TkKavPNQDeSlmEYZH/1/XhwIE5E7ZeRdkdR/8z/uWyr3/SHLZed\nG4B5iNJj4BpoXGmVO+wpeMmR/yIbC3/4CrbpLyN5B8M7t+hPBQFnF5EJl+pM\naLwIErjFH6Cv/JWh8dxuaMiUu4u3303pkXU20ZGEri29pb0AToa7wqTKuSC2\n8epG2Lm9mzKsTAnSSMGnhmu6vV87q+PUJWRte4n289abt+4E8M7K2UqVGjw9\nZUzFbRnarmdhR+UWMZRcVH+Yix0ioIC3vgh6rsvRCCoyTy+ptfOaOccN6nTJ\nrsQg43SfXZ1UaspEh+upgKRZWbhY05585zs+1DWRsOKYOujzYn8KGYE/Fs7A\nX2FzUu+c3k1CJQ5gK/ud/lIPY1X8ACn3oGGuswSSGaWEtymjokTEldrzl1aG\nk4tF6T/6b3OPQ56rbgNr51TALcijgg1ghhnz+9kq837KrmcWnAGbqiYw8Yuq\nrSad\r\n=OsVb\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"8ee5642fb655b1f67d9b7780b06cf77660ca3240","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.0.1","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.4","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.4_1611973749805_0.9140211328352825","host":"s3://npm-registry-packages"}},"0.30.0-canary.5":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.5","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"333df630a2cc6ceb9fb7a1fe76a23a03b1c8a2d0","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.5.tgz","fileCount":28,"integrity":"sha512-tc0+0U0Pwo10u2tD/rFjxzqfLEl389TYWP5X32D4oxOKYnQrABsNYeuzK8NN4jmjoGvMZlp/cNbeqckgaTjJhw==","signatures":[{"sig":"MEUCIHUljoBH56c8qHNmTmFjFG75Yl1mctZu3pPcQcPcmK2HAiEAtTa05ykk76W8phup+aFNZYUbp+A+WLgpyu+7cSge7gk=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416226,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgHa/3CRA9TVsSAnZWagAAtfEP/RdHo/EJ2c3b0STxUyVm\nz1r7j6iJ//czQlB5BDi/eoFOVWGxOO74d7js/Ob0FNVXW37wBKhUNCGBS/6O\nYM+pfIB/5D3Z3FtJ/VfL0b47ZhDIYdgsJ+yxfWzj/w3JGm/C+whneJTVQNJe\nFv3JUn5GfuGYc6GWKh9W7KkJ+ZsIKqyC6+x3fFchgvrQntpK9chwvpyFoxO0\na8U+hq7go3Hx1iHPPw+g04J3aPVY533RfLyz+lda8M5PPD7HbnsQNznSIUAa\n2cGs8NptPCbJhaSj6ybAOYbJ0vcMxMkMVzMFPih41fl25zR7TLH4BW24cx7G\neaMP01jHFyInjggzljc3wCt6AZ6FlqldBIZX0784EUvRjIS5GRKq+Z2HirSJ\nLs9e44sYn1VpG8it15mcAmyG0Lq/QmhASnckO79PzdDx3uPC9LGYySt/ynJM\nNzVHX/cRchKyMXyK2sNkEB/Mo59nXm8c0nG3yjeuAKd9ZxTjrMm4f76BMrbx\nnuaPtnh6bu3Eb4HMncZclNMhjz9P8wDgK7UosKOrnG4BE9icwoTCVFq5HEjY\nTyrLcP3fgJ3vAbb69EnVCLo46DCO4DQW98ffw5flNahmF4esRVe0qAoE/fpZ\nB1ZyZEzA/US1ZkdvvZy9BNqlEgO+iWFfPWnVIjMAplEejoUVSTFjnygCqoOt\nQxuC\r\n=DVwA\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"a0d737864210740981a8de39710400f7b1fda280","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.5","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.5_1612558326791_0.5250188755965706","host":"s3://npm-registry-packages"}},"0.30.0-canary.6":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.6","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"merelinguist","email":"dylan@brookes.net"},{"name":"ryardley","email":"contact@rudiyardley.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"bac5847bcc33cea42b6093ca714573681e91d032","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.6.tgz","fileCount":28,"integrity":"sha512-aRzQ38O/7M9XuxKHlF++E7Io6l70LcKgRkpEnu3kt6nowdG+cOaxFTsjjdGTFHSWOwsV60dyBoWNLfRjzHymJg==","signatures":[{"sig":"MEUCICicUiqQPCRkMbP6HV8uAZoGtPJeuAmP8xeBCnahxZULAiEAnW1Nk5cJJ4Pvp5nU64bnFB2z7MQZSvmG9B6ZfjdZo7Q=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416226,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgHy8TCRA9TVsSAnZWagAA3N8QAI0RTrZ46+MKHxwB+dem\n/1b079ebm1oiK4vH93bP973QAnXhhkizQ5B3dWmQpaYU2nfJwdXPdtKLB0j2\n1ZZRNe+nPDqO0JXAdeiRil+xm6gVVyE2Nc2j0O6Yt4TW/C7CwXbftJjaCUch\nsH5EQOrr1o+m8eA6qbII5gtkc9AioPTRKVoHuiHwKEk/SyWfR8F3BMHWSt+m\n6Y9WBowGPm7dD2XwEQY09gpAQJ8OCf4bg5sRgRr7kIBI0aO7pAftygVnbFB7\nsQz9wOO9s70kvejXxnXXZCiCMG7HHYBA/Lz23gB/Q0aEeR+OSteJUHDdsa7K\nIuhE8P417nCtpkDYrkMbGiS1ss5K1No5+yniPnY2wR8GTI+eALnvH4kTX59e\n3KP1OuF+bUgs81K5zrHfC9/JqfuPCI/FMWHdhrXXxWEYbH6batM9gm22TYUh\noVez5azMTnn5DDJatgR/Ta8RmxXH0hOFiynWnJ22iafCMde9WT1Ij2AbrtGk\nXyCULB6cBe0P9Hfar1es3yA1U+EsnV4K71oYp0FvTNNl+A3R7m65tCdokd8x\nazKmvbqaOMSA2SixCqxFoDWfaDs7XlIH3A8n1HXoSVZ3nHS5F9o0eXOjjDRH\nqZRVR3BIl3EC05U+owrjLPHWzBuR++4NARfBWjG1JnEBYbrqg1XYi6xt6oME\nQl4O\r\n=6JTa\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"da6393c5389d245a81390912608b1491ce8208a1","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.6","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.6_1612656403466_0.2633268110582352","host":"s3://npm-registry-packages"}},"0.30.0-danger.4b23a38d.5":{"name":"@blitzjs/file-pipeline","version":"0.30.0-danger.4b23a38d.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-danger.4b23a38d.5","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"bc1d375bc9e5d225f2131572e78af9241c721135","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-danger.4b23a38d.5.tgz","fileCount":28,"integrity":"sha512-i+EQ4m2jIv0EU1fWcnEVU1B7Yn0TahfPa5yEqMic81w+tRiE1HK15+mRPsQasoejHP8lI5StY27yCniyLfvGzQ==","signatures":[{"sig":"MEUCIAZe+OxG0XH3i4ZxtJMeudTR72oYLnyV1Eae8HKTKd1MAiEAsYy+OX1fn2tztlQ3/WB7h5GNQFfZm1EIffMpvDXPzCc=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416262,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgIwwxCRA9TVsSAnZWagAAfMkP/R4r26dd18mCJHs548jR\nUhKlzDONlF7qeSYaBFQJ4CUGUyNqB74pkXbzmmG3Np1OlRdObXkvs5knn4Nq\nzjPeboSSyrcixn/Hadja/sp8iq0Uq/b0oWAd0IgdUAD/AWEH6XSloG2uqoLV\nl+ILY9XLEcnd+M8XiAW1XvkqGtBZfp3Kec8/l71uJgL8fMehsnfmEvgXNKYD\n3ypuFftXLP4FRjeR7KHRdmDFt+LWIE3tArsFDDzRPz9c4c4+pFKutlnAW78i\nMX1p96YvQx2VnnhCPvyZDlRkP+iOpssINzDM8YbLdmLN6lx7oDfB4nwu4KDc\nhyCjXUgxQiMhjZ8aFwgZcCOmbPI8fxfvWirkjIuP+siXuSZRueMe1umDhP2F\nIZqTgnZ8sr4rbUWiM6GwtlkHavY0YSG6I0FMcXLPISAvFE+FXGiOfKrhkvQ9\nkWIsx46cYle8YdwVhfHEnBgw0wFEN2ok1QLyBD2wpf1eItYs0JCfZwgKKbN1\nZINPlM9h44krogsNkhwJ111kMTRGKSGaQjHxOFuNQXcB/GQ0klvSQrCKO1H9\nPdNCCtqFE41HKZoa8NQgYTNVBr/NSKUtPZ94MdTO+UURQYbpumWFWpAnDyU6\nJUEiLBR6ixf052glUOh+CkAlbW86O8hvLljoMINY3VaephsGMQiD6m4qOFDj\ngopa\r\n=Fe0C\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"4b23a38d30f191790ff562ac27a01ee50ca59b70","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-danger.4b23a38d.5+4b23a38d","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-danger.4b23a38d.5_1612909616997_0.2689848346682555","host":"s3://npm-registry-packages"}},"0.30.0-danger.62a62ee1.6":{"name":"@blitzjs/file-pipeline","version":"0.30.0-danger.62a62ee1.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-danger.62a62ee1.6","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"7d63dcc85108fb22b66e24093b7f5e288d0cb0cf","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-danger.62a62ee1.6.tgz","fileCount":28,"integrity":"sha512-gKFAL2ktdXskF6WtPq5pY1RxhViiI41QpWBMIEWyk35ICrz5bwSw2Vi4cYVmJPJbYW8VIETusvhWlZ+q7I7i2A==","signatures":[{"sig":"MEYCIQCytWDQVkGrrtU1U//nY+i8m2Pb/GDHI2HEaA9yGSgbAAIhAMfCgqfxmzne/QvF9oAQ9BBgOm+IWaTnBgFErqdBK6bq","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416262,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgIxJoCRA9TVsSAnZWagAAm4kP/A6E0mmd3lQHpZsIEEuY\nzITLhQJXtBCpZmzgfkG2tQJNq572O21h/RtXGttp+YAngxxomOqC8yRECQeq\nwyVULmcY9WxC4EreQdxaIKRsCvkPngude/uU4w7kPCe4lHhapEktRWLeOdf5\nzoUrelaTHBHH8a8QNnwBv8bHKW46Q63py2NHFVASPhd5dtb5/Heruwc15VYk\nG12r2aUXxtHvE3yP7mrboCiyu5D5yTR8fdjNKXbRYIbyvk/tedyUAkFbdDiz\nSRdgpDyTpT3D4aPB3tk/QpCGQ825tWOfeKSadqeqkzGITQm3NNLUJyE2BV2e\nOdbYnwpu4z7WIM7V+NkmwuswJ+WCoHb519nm84HxWzTiVXb8U+sgRHlI1Mei\n/yypZ5q0/tWb+J16d4ohONW6Ps2EvWxp/VJpnTSp4G8z2HRmKz9RlgFhJrP/\nTxUDfmpt+Gi5DVj/p/l7YAHtpp8ISbEWhliIYhLmqWQfGqVyDKzrth2s5ug0\nwXsJIrnSCfjU/jUd7hYpS45nZ4DG3UC0EKIVREnzIn2qeXQLr9R3DatJ/DxD\n4B+qdmNvejZC3nRi79yRE4iNMJ8dbXrO0v/kdmtyBIaelT3peNxzQH+M2RP8\nn9/nBGlrKERp4RwjpkLASUPp6hGBO6HvzSVUuYZ2j7vsmth4tY906rTp0q7a\nNobN\r\n=GSdS\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"62a62ee18c5c2334eeb8ffe7a09171b91dbbf95b","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-danger.62a62ee1.6+62a62ee1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-danger.62a62ee1.6_1612911208152_0.7923574811227869","host":"s3://npm-registry-packages"}},"0.30.0-danger.bc7cfa45.7":{"name":"@blitzjs/file-pipeline","version":"0.30.0-danger.bc7cfa45.7","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-danger.bc7cfa45.7","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"f88c062c208c31b7720abd3ca172e9453f6c8144","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-danger.bc7cfa45.7.tgz","fileCount":28,"integrity":"sha512-M0Vve2flsNnWvOb9wfECWl6cHEiN/w+mebQB5LMwVmEb1Bl2ueGOxbZngbZFdpOIJocduUXdJZSAAoaPqBXkiw==","signatures":[{"sig":"MEUCIQDIizC0rm6N43mIdsuEpT3lxiETXCAIv8hoOkNlWDC0EAIgArC201BYJvU741uiuMLIiWl58GzP3fjzozbqJKZrBek=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416262,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgIxO9CRA9TVsSAnZWagAAu1YQAJqGVi+LyxuR1rz3y/V7\nvU4wqcZwJ29ld3SNFso+7T+g7UetFHifsZoIwWr9KmmKz2EJ4PvSi/YrSHpx\nCEtd6XRchA/EbQ7SrQRMKCGFyaIBDUF2Oqa74Pf7JavYIxWV4fwkO7cWmapt\n82j6znnG5n2ALyhy8Os4BuUhtMmnt/T7zGPEJ6NvalkAo43r+qR3fNBLoA65\nS58EIyemaY9him/N+YHbz4b1DIJEBQexJTDC3bvGTglg0pWnwzdIa5urv3Io\nwwKRAzOfdzmnMpdJyfDuUlI9dGdQ29QC2Bpwjq6CK84vInl7S0q8pTuUlpFT\nE5PuQ17EsagAIVq10me0+3asHJbz8YatDHRp0HHlNt6XhehiNAHRkNm1SZm+\nVS+2JGCjUvuGvTIbfoITlIImS60rq44/7EnOxBvJMWEGKD9lJ3AgxEi3e+Ji\nE/dI8RUdtw09EYi1ynI3m29G3c8QPnP/pQCkwpU0jqe3gfe8IpOF/s0/co6U\nLlbsnTlhp5gTwomiVekd6Fy7HCjet786to3Gn5VlbbVcL7b5ZjknpdLQhPpm\nklQILzHbgmfHyF36pm8WQ+VJ43bT+fP5Ymk3MjFQee94TpVDIWWpwsQQs5FK\n5sagWe1j9yJbxH6jxInDKQrEFWu2cjQ6OjDr78IT1DvbPe22hrQ5FJEcTIAu\niR2x\r\n=MedP\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"bc7cfa4531afe398cbd6fb2a86445c7f839d6c33","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-danger.bc7cfa45.7+bc7cfa45","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-danger.bc7cfa45.7_1612911549309_0.2608573310319351","host":"s3://npm-registry-packages"}},"0.30.0-danger.15c47545.8":{"name":"@blitzjs/file-pipeline","version":"0.30.0-danger.15c47545.8","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-danger.15c47545.8","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"bc59d3e29058a1ca7c9fa8f557d97a5313d80dd9","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-danger.15c47545.8.tgz","fileCount":28,"integrity":"sha512-siq4r9ck4rcSLLFC0iz0oZYNciWNPj6a87e7I9PBrn8yvuaoTHyGTx0Dk0+4jzPB3wQ7UPhQKYkvK1IQyf4fjw==","signatures":[{"sig":"MEUCIC07nU2DBk/T50cmKcKNGF6R49gYrJ0I1bGDUB7Q4CP4AiEAzRA7psiYLtTk4fZFRsRiS/0lGww+C4JUOMelFlC0KlQ=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416262,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgIxVzCRA9TVsSAnZWagAAJy8P/3ie8xuarTSj6/1dE0I1\no7z/QZHxzjus//o8S0vRG6990ZPQdkWeBirSxKMszA5w5cJ0ZueHUakvTPfD\n7ahLAed5CyObhMcQFpvPRKZWP7Qkzxx2G2hnp198bgPOMSmDM8oQQC9MjYhs\nY/dYHrFZxslJGZJc/uC7OJFp1DxdX+HIcXXMwnCtjiPQoKxayLyjwTb55po6\nxQprbH2vHZYTrGbrTvMSM7wIedohlv3sQ2vHZiLWeP0XTeP/vhA24N3G83R2\noFoL5yrHiMsHz4zYIqB3MbwOEvayR91Azu/bRxKbNbfYOHB0P5d0mk36DhcD\nM05y9zwoF3ydCdMEq/XK0CqsilX/m4fKC+kSBwM461wQgQLFzPI4n0aOxidN\nB+XSI2ZaVoNS8ZGkPJBd9Om28wS3baMNYtWltsmIq9/+FxlqiUXgva6Z7GLd\n1u4pgm8Cu7gqLjl7pbemQBehbzXgj2aie2X4Apoz6O97EWRNiflzWKfOTJj9\nkysrRm3qS5egksIFGaFMonyL74rR+CxerMfreYnt+L8TzwCa97fm3/i9zNwW\ndeNPRsLW5r0kmY9GGq/g8VYt66RAycLix09nFpUunIIDRPI3n9GJLD/jcJC1\nQQodvYnLqdGiBywHuqjaAs2Ufzu357vpdgYNVTKEHvo1JaXpi7FGLES++TR9\n8+Nl\r\n=+qkV\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"15c4754565f833bbb75022efd27bfcbf330b616c","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-danger.15c47545.8+15c47545","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-danger.15c47545.8_1612911987325_0.13168321245829362","host":"s3://npm-registry-packages"}},"0.30.0-canary.7":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.7","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.7","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"bcc2e4337a237b8fc03ff255f6940b050c215df7","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.7.tgz","fileCount":28,"integrity":"sha512-EZQcAh71KKj4Zx94R/exi0uGbKe2RtdiPAxa5Zzv0LolWnhOl3y6hb4XOvoai49m9YpU3dXtMTx9tr7Aa27I1g==","signatures":[{"sig":"MEQCIHwIOR4fPhyIkx9rqwCgOIHTWy5s3RsCldbtnm0bijJ9AiAo3dlHjC5v/pZOhmB5eMQLU21DBrXq25K1sjcJ4aSrXw==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416226,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgIyqvCRA9TVsSAnZWagAA/VkP/1nbYfyz9/hgSrzxzPsD\n+6S2nsi7RC3rRqLCIjYThFg90+bPTzJYbkAWFxpflTZsXody4vr1faD+j8uP\nxx87XNKqV9rhCvXOVyqPl/1TyqkcTBjm2+cmH57ZUTZKbMMxsx+YoyM1W0dp\nBUz7FJ52WJEx06F/Z62dXDHlyt6HwwnWN72T9eNEhswXF08S/gWSx9mSMBcg\nYhTPq44grO14lUmxrvI9zbZjtuZgP/y5tk15hrWz9eD3mYMA0bZPBC1z5z4+\nl+jvcmP0FKdJ0HPurbTJXNf0TL1VK6okucLwA7klzSeTRU/D3AlwO24lylu9\n25j9UXPTw51IOmcpRPOft6RM35T2UXcqsFyWiwD/Mjk3hmgqHQokVXaUlR/5\ncy58esT3cx5ri4Hvx1L4TS857u7N7yQAYaL4fNgBbuixc9dC8MEl3DgMPaKd\n+wLltkKB7j6fBMQd79lXom7pt+4mgTpUG2jpF7NBZ25jz/CM65QT4ny3zmoa\nMKG4T2FJeJ7ZCMfUPSv5oFxMnNtBBwVWmCprlq+ma+Hec9T2WvcE0xXqbUxx\nmpayArZuW+Em6ODG/1KizHcTY6gL1psa+7Ve8zBm9WK/XQC25Do+cgOSqHbx\ngP+0GfzLJ5LvWriFaltppPrY9sB/H0etEsNF79IF/UIlDOHn1CIIdYxRGvQe\ncl7v\r\n=ttIk\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"1792cc678895e99b3da208efad15c3d5d3a50fa3","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.7","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.7_1612917422887_0.436587802700954","host":"s3://npm-registry-packages"}},"0.30.0-canary.8":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.8","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.8","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c581c2131663118ca6f42b66f5f2819dffc64a09","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.8.tgz","fileCount":28,"integrity":"sha512-E2D0FZD+LaXJJluy92/iE6Gngv+UGerWe8GVGLi1/NsYrEiYLERK+zKB81kDzZ0TbhnagAv0Q38EARGqBxhi6Q==","signatures":[{"sig":"MEYCIQDXGCrzkd3vX31wV7TsBy9/qglx7IiF5wrrjUT/PkTxnQIhAIVelmqWYglH+UulEIYC7RnMNrMfF/mrsYmeS1VxnAwL","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416226,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgJHjeCRA9TVsSAnZWagAAivYP/17bIIbJJ653rRbcQ4qq\nfut2C16Qih8pguCb0zCZsrvHV3JZQPKpl5UIBj1PmtHMpxZVKG9GbZzPNO1s\np60rH/+1sCHWrpKrWMFE7mJQQHVm9GcGdWXjAjtswqV88Su2OijWWsBK2yHK\nSXNWO1w6eqQSeuSYRXVOuyfySM69IeNr6/ZqmNVRw6Us8upqt44xhC96+M9s\n/JAJM7p79DFgmXNLdpeQTCNAl8MSy/u0hpILkmGJUOdRIRvgbRzkSkSGiHUR\n9iKjDco/nbX5dXkZPCtMwj5s4i4WXcISc3X2kOkN2H88kW7dDTrn3t3z5EhV\nDs+b/h5ufDwR8RI02jBNGTj7V9YgPRWPctmebrcVk0MU/o9tbJWM9kqLQ845\nbqybJOurvV7IYM+2i0Cuw1tQiXW3KSpdPnIOJwAqL/zBOkyZqQyCMPlFQMv+\nWSY4XQiS5qBDsiJtB+N0xenFmLLbzjoOTkH6y90V0oKuxBPAo8YvcxnEs7lw\nnbalWq4RMiL1qf9cGHKO3LUR0a0SSQTAkuLPB7sFAyrRHSMiUI9qvkpJgR33\nbhPwqj0YDmEnQsea+p17vGGMNt+nNrHhfi3MWadkve5wKtgC20s+vGXlAmrO\nvNUnbS6ElGFugBk4YnxEkrkqLc9/fn/y2XZTlAnxcgi6RI4ZB25O7kwkKqjB\nQwkw\r\n=IIkd\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"e2383fbb5f52a59042640dcf4cb67d2c8344de06","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.8","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.8_1613002974476_0.3777989558534758","host":"s3://npm-registry-packages"}},"0.30.0-canary.9":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.9","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.9","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"3cb8f37116af6cb60e1c5b3bf7a2bb4939bdd3af","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.9.tgz","fileCount":28,"integrity":"sha512-eQ+Mv2YATTMZ5Hj0D077hcFodexwNaKpCzLF9SZfKRlpjR0gykYo9+Xt7rhbNjnRA6voaibEI0Jpn5EjMG6MSw==","signatures":[{"sig":"MEQCIBcajOPYxUY7Ba0KrQ4MD8I4s3sG6cR0vx2ow8e+q+C2AiBWBb5tApTv2NkkwVYEX/yr2Zglo6IncCSDR0Xg/GGLVA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416226,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgJZomCRA9TVsSAnZWagAAROYP/AwQQtmFkwSSfvh23b7J\nLYKMqsODVHWRur3MxGy5bvUdXQVvqpQ0RKMPCg9YXQfj51v1CzO+IGc7LFml\nAc+YvM57DpIDZ/jd3aFFGwvimv5PujiUSm14Bd+3T7S7iq4kxAHocYfcJabU\nuqnbeLcan3a2ghOEdIcqK5Oue4SOyiCR2JnPEndvlT9wOoWC8GoOwwX6VWXN\nHEycp/OrvGOayDTGENbGR/NYFujDJWBxLbX339hrDhjfFRVNBSgQD+2O7vRC\nwrFPIrxuMBegwIqjH8txhW6JWqzVxqlhXbtyZ93TUEDrIQRh5nD3XM5MPXZV\nGiysQneAAjlKpFS5LXAzZG7cI4Ggxv4JPvcWbIMBw94egUHpT+5ZSlfIySxD\nulziKE/at0Z7HcsmrJaea3YZVJVBI9mMRdqcvcqGuQ4nqBNRqmaJ2hqBCUcv\nLQk6am99YMW8XpC704evtc3DU/NwkjLKIsOAvCOXyx/rY2IegmGsdAFk6/ep\nOJ32G7nJfRUe1w00Y7xYttgssQgh1DuwNbSEMNrQHAzUytrPiK2YQgzLxwUg\nMMoGIyRtxBzE3rsThfxHG8sQgUE4L3Q52gKd/q7fDKHgCYJf7M9uf+122vg8\nqh2rp1ivlbFs5fIVb7/r1d9t5ZEuDtFg0rIvNY2fwYckwy7/mj17a4EJF6Kk\n/s4F\r\n=nMke\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"04b0cd356cab81795d1e8dc7d3cec9e2610f1bb6","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.9","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.9_1613077030207_0.045932250373011074","host":"s3://npm-registry-packages"}},"0.30.0-canary.10":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.10","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.10","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"b7067669ef76a72d05ca0781877adb4d23ae5f3e","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.10.tgz","fileCount":28,"integrity":"sha512-p09cSXTr7rstyZ/c/hN/0SA9KJi9V1GSmn4WEA9es7pxdJHvzkExYmdvPc5bbEmxj5MROO8yEepv4QdOfuhrlg==","signatures":[{"sig":"MEUCIQDDUX+DREC0anSoLPSk2kaRDtvnkr/WXfVf/HvgekxvkgIgY2q3DVi16blzMbmhvqBht2L/l54srT159kbcQeqmTfw=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416228,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgJa0JCRA9TVsSAnZWagAAyN8P+wWx4slpxeJUTqUyV/fd\nyCRBgyQZCw5kM+JLq0NQAI1ldzgcNz47FODZ4A2M1Pfj5yBbmNlj7lu4gma+\niwWCMrLiUjHphph00IpxFxMFVtOWgWK5jVop4chPhSDSivBhbVfKOIZ6OAt8\nSwzPBzu+XUUtOScqsF0VJs7o/abLJJdS+sYTQ35Yd5G2DLxqSFtY2CCEMrqx\nPsQvN+/OBtjBrdrE+5IL/MlAY5us4ZMTwKzttlXMUiI6VMUN3FdldXovHz+2\nswqb2MtTtDs8keQjFJu5i1MiYSeYmeMrdyjgwnSuuhV0SVa5d5ymiR7K3wQE\nzcj+pn/ySRdd91ig35SHNWICfny73Yis8LzkZ2sxD32gIoTxeOz1mpqkNycx\n1cBARdzICOTpCjmEoSvFBABFhvEskwYKjd8AQV1S0zNiXqq4m6DS7MCfoRFt\nfWcEHQgTk6Yrly8QLgKkykjIULSiz0am+pDssu12Pns5QjoaHu1C0ua9Yi9+\n4ly7F2HWZpdQHmLEZPQh4hzami1BtIM7HrW7/FIDLazEaY88ObRZ5lG1icZt\nyU3TmdnfnCZ+jdMWasmhjb+sE/UHr4+OO/V/kL50KNyrox4Vrm/xE5bEf5sA\nW2Zhj62H6NVhjDD22u43EULYGY4AlY4SDHUYS5dHmzxFlmV/71GygAS/Vlds\nA1GM\r\n=LeS4\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"a4320fd97406dc36fcd418fcb3a2ec858cee5dc6","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.10","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.10_1613081864984_0.7694199961197998","host":"s3://npm-registry-packages"}},"0.30.0-danger.c389a7ad.0":{"name":"@blitzjs/file-pipeline","version":"0.30.0-danger.c389a7ad.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-danger.c389a7ad.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"3332f19be0697328826742f97f0f437272926556","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-danger.c389a7ad.0.tgz","fileCount":28,"integrity":"sha512-U5AkFPL/MA3qQzQoJsUy2VRO5oUBCKyZsmlID0hY/Xxy21VqQJsH82NqTGfhJWBhlnJJN0/L43hrCBMIiaG4MA==","signatures":[{"sig":"MEQCIHm44kYQwFF3YyqpB1DxpTTjUYDLEr3nYA9RLSvEegfHAiBWHC2CMWTHS6/hAGbA/ZMylKXJuq+CoFTiYuEQdtXC9w==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416262,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgJcKoCRA9TVsSAnZWagAAxlMQAJ8CDboEGwGq1e6LwzRl\nxtaMuGsVvlG/OaCZItUDq1u2oWfGjBKYCu9DYGKDOjW4hoWJEOOZ//UyLpAE\nW9lBvc4zU6XWQaRGSYg77/rYSdV1EuepLdrSeQe57KKFtV0kWZscJAeW4JJr\nRnsAqbuas/Q4Cfr3ohtjlfBKJAV6UUDZTnJFduZzpyURep+YmzR/J1MPb+79\nYDzJmZzt4er66LhJgkFKltXaAR7KzQah/iTJ9wXRMEM1IbjmgUJFEm/EjSVP\nBu5HekLrUKZ70o7rCOY+T2yXJA6znsc71acCgUCkh9vgo48e0AL/U/j8f12X\n9UEmCucx04SkpWlOcSd1/TV1OMXBklllg+wjctf0SMGHpCiXVdT4EddEF/ld\n2G0SdXIHbqXzBaNfTATRygyAYJpqO2HYBpFRHb2JY0vgBDxY2Gr0nGlUa9Ut\nc1ic9KI2cFvMXRRjs0pxH3N3qjlC205BPs6SqDjp+0ZMQXba+6499opZcDMV\nRZVDwzjUHMaSbRK54ZqAm9k3JzdGjdequtah0LPj5yZAoyZsvFT+q5qSmELK\nVFHZc3wi0TXLCpo8XOMAA1Fgweo2rQ2UQdM6kfXuBN+SPkkMTFiFXiEQ89Rn\nKPvQ4VYqWbTGy3PxKzWDmDC9rQp3CNBMhiibPep5yiL0v1elwLQItaL+Z8ax\noPil\r\n=HHg1\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"c389a7ad6b97368b34d61f85912fbeb61e78ee3b","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-danger.c389a7ad.0+c389a7ad","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-danger.c389a7ad.0_1613087400478_0.03926894115733104","host":"s3://npm-registry-packages"}},"0.30.0-canary.11":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.11","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.11","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"9b2344bb71d848f6a1eb40a14d65f29c555ca499","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.11.tgz","fileCount":28,"integrity":"sha512-XQptam31k6y7xavcDvImNwf4kJGkZ6H816g2914QIApakMxurly2X7hAPuG5vUQoLEtm0aQvyOXQCIiDp733+Q==","signatures":[{"sig":"MEYCIQCn6ihtNJDOYDq29mcF0BSfHu9plPNbepzBQDgQXH9lGwIhAOGb8/fHzYmEceWaeQRofcpJBYzqhvtFvgfAn4zAo+2r","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416228,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgJdHkCRA9TVsSAnZWagAAR/cP/jEVFl72A8DndUut3t12\nfndLZQl4Nv6e1SI+cWtKqqiwHKDAIn7XrmqZ+xFvwbBjGDgXRylEZZ7NTCau\nwIQ/BWqdRWvAvi3g9LZmVNrNfQvJSRaZFNmLdi4GMDA1OzUv5GmzBpBgNbK0\njNzA/m6oxoUqwpb/haHDyIn6LHFRL0OInd523Ochiu+Il5//sgTHGDjxMUy5\n0ctEulTcgEU9o7IZeHalFjqem6sPVlNKWSJmrpjz6Sts4BWDsvuyRK3s/9OT\ndOx1tR6LsLTfdqsxXm5dpaIZO4nE+YNMldl1qj6fOfOUrCuIL8TFshOWjnIR\nRpxPIRd1J3tS9A09zg6YVFHu7b/m8BHcBdVADz4h6KjwvFnvIADS9a8pbPwi\nPk3K7DUez/yqNJWX3j516QGbkoPoTrsms6afwAVipz9GAqQDfgxLysJxezzF\nadzrvGZC4trhkHaU53cm4ucn/cDb2nkseZwqGxOyYxenlvrFBgXCOtCnRJKf\ny44H/rXM3e88bmWig3X3t0seI3Kn5zjJu4ztSiGliXeByxy8r1ufupryj/Dt\njV9GOOpqkY02g8t1/ARYQcnhA27I7dbCLtDmba/cjppF9VeZEfwkoh1B1qvr\nVJwjWrlo8HCRXpR4d2Xg9q6in4iG1b0Wos0mx3i1V8nQv3Q37dYkLptteTVT\njd2R\r\n=gJJU\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"984de2cab17631b4d7e2a01de3a28126b75b8db9","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.11","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.11_1613091299741_0.05685883276245218","host":"s3://npm-registry-packages"}},"0.30.0-canary.12":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.12","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.12","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"b12bf4fb37301720ba49f2d62794c4c52188d2c1","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.12.tgz","fileCount":28,"integrity":"sha512-wqJVG4BU3SkPbBLyC+hve4a8Y3861dFBMImKEFlEF/CY8oArNFJA4WNPI/uRwbv7P3eiigfyJ13XMTYqmeZg4Q==","signatures":[{"sig":"MEUCICkWNhgH2hqGmhfue3IS26AzVDXIv+bWxh6wH16cuoL0AiEAtpc+LWql9+mYKBHmPm29i2LpyPOGvyAwagco7PRK17M=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":416228,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgJtK7CRA9TVsSAnZWagAAcwMP/0ArVbKi87IUWscdKCEx\nIKAhdGPHRiI+6SAXU1m2y1iB+pT/4CrpuR3Ob8hx5bBSCcWxmgWG9zGOvD0X\nNWrj3M045GySw1V4bHskDgHL8yrgVFJM/ZT5rCqity4xB+Kg34h7yp1RRyr/\nhS9JVqOZzGuoxYihXh5Zvdh3f1m1KwCV2HRxu18NKw16n42+waciG1sMZ58H\nesTw3Tuw2sp/QYw1z2x+rnJsLdqL4dAy//MrGkOPQj5NejzX6KsFM3UUoH1U\nwIKNLXdZtGlCphFAw9TGukqbYwK+VlLgN5AmI7aAjBVQ6JpkDfnrdinc3wfI\nJwwsf7iEBa18s80/fa3wRefOkvjKD5bVyIMRAnfxMRf5nG12eLI9790paciV\n55E1tRG3hoofJxtUMvBWdQAyuMGGfNeoDMEZQYdxwq4vZlumau7lb13hScgg\nGHi2Jc3HC+br+Z/YwXs9uRIsjjvl2tw7BSLcW9Yrd+u0tpCc/0bIi4yM2s+C\nElAMVAteOE9rCwQF3nZjWYoUSJZ3XZ+GyMPePCxGym+ahcfjXTLIyXzafwlO\ny97ClNQ/OZ8Ivl+rKM57ME9wdLf8tTXfBxABAI9P2u0lDC4861YmLfc9zM4K\n7jcAwWGn5UIqzX4k910wl4dbSEdYRdoqBLWSKRHJx78r8lDnwqVJvXHdldwc\n7Go8\r\n=Hmyh\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"57f1e7d70318ea1a942e4f1995692c58010d965d","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.12","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.12_1613157051208_0.8443913070734379","host":"s3://npm-registry-packages"}},"0.30.0-canary.13":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.13","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.13","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"ccada1f6904c1d0774981cb3ac96bc181a2fa52d","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.13.tgz","fileCount":29,"integrity":"sha512-ZghWRJA5ssDMylK87fm0MkCIWCFsVQt4E+PwZOkkgHtkjf7sFDn3lI47syn++eYrBBdm66JRS2GObsFalbEicA==","signatures":[{"sig":"MEYCIQDzYTNnnyEYqSr82tol8qjQLLUtfoMfQ3Wbo5eAKKh5owIhALj32Kf05a0de5ig+S2pb/07BDbhVn1q1vt2Q5ibBzj5","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415055,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgKIoVCRA9TVsSAnZWagAAxSAP/3SOJ+GRbuMxgzeTHZIr\nKvvJfZz6YWLHQ1xJLjMSZRBBAzdDrLP4ZwZw3ZRMDE7RUF/dPN28tO9ugwbE\n0h5AYwNhIk/h974bv/f+1wnqL3eSSeeQiyoKC2hSwhsg4RkWnUh2euhMuNDJ\nuyhzFJz9HZoMch/wSdsQ0ntuXrHFLRl2y8pygULASBsbUlty8UEepAsF4CZu\n4aqY6YHJea2Dyhq6wM1JTSU6L4jCpJuOWRM4N4Dj42WtwJ3i9yLEa0K9J9WA\nWdo25lekdmR46HqDmUlGNdpYc2xX6tmc2ZWELR9q7vmO3BXzoN3+n2IedU4C\nt8Q544hSEUqCUKh3IUrgfMUFjjPk0KitYuKUP0gbFmszUQkL7BOg6G9wFNBH\nZludJjbA7Aqr3hAE2g2DmDiHiIFaKs70OMH7D6Ilj+0jE9JBNygg3pqXCWdB\nwiMf6sIxGEWI8KfUxZ/wleb/YZ1dUEJp3UCkoc1RyFnKqBBBcNk/nZJNHHOk\nzEat5UqZS9qAyEwTmlJMm9AVD8vTWZSTXjroAiZxkKW8vqatxh1He+Gh6Prt\nrziGLLqt7312pJH+HLEDT0W+s0B4NHetKhYihz+LmuBnk9uwtabAgrx5SIvc\nbNc6XzF2KxbzYeQacCGszyloW5RszNAl0PPhBQeY/rZl67G+07OBFWlaXGBW\n+B8L\r\n=cTsy\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"2a25bab41c99b704275ae93af4bd73b6b0d803b2","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","rimraf":"3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.13","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.13_1613269525301_0.5612975625091874","host":"s3://npm-registry-packages"}},"0.30.0-canary.14":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.14","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.14","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"e25ba1f42f5e4d7ffe52ef4cdc8c172c9d01bc88","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.14.tgz","fileCount":29,"integrity":"sha512-Ck3BPxF0hivbnNgQ01J3sqzwKtOJcp/D8COkDA/YfRVSef9PzYtsPA5siwd3AhsTUewg7ut8PU1+537GzVRn8g==","signatures":[{"sig":"MEYCIQCpaZpltxv5ybQNvCQIwAGXzK7EqXdPoyOu1+mqxIzi9gIhAOVpAb2X8xaVOG2FxNxE7sBkwCdRHKrmUctZisKF+CQh","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415055,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgKsyPCRA9TVsSAnZWagAANSYP/RZyxcu9kf+FbziJUurW\nWjtu9qVZEbnLQKAWlcleEOUF9blgiFFhTVKjD6THXWqYposwUMYDHmND2z1L\nno9aqUmA4jtuogJC7j722tC0BskyOgpsjXIaAMGWHvePzqqVRki1jD/bU/2R\ns+AoegSEQtImaJ1S25EzIMyvECbVOqhJCj1zEBkdpR6SHmb/mu9AY44qunPT\nMgOhjGFeNFbdA5RbtpFZT0cpqN4uHGuynmYdIviau3SWzeq7Ia2qjBghzrrI\nF8M4lMW0/U0vJSx01YVlivGW4zjOCrCm24SpFhnKC412OxVDCfzVci06E88D\n6k668UcijyV0/xXoTHWi2wtJgxpOJAuKTjaXBlFmHL47751PBWlV3Z+l2cKJ\nUBY8Ju/KM4DoyYiK6OJ4AjGFYRMX5w47I/iq+ObmClSdQVR2U7+UYlt+bqYg\nD4c+VMp6zZT5Vb6UNe/WH8kubH/WND5ajF5Fghh/TvHSJRU0zjdiAq9en3bf\nWBDIEFAjH8OlfLh0oRgewWDOt/8W1WfUAs/r+PO7kdcMA7ced943G6WgPMcB\nFctn9JofhpHyP8qO9RLGcz72hmAV3GfDuX3FXCd6fEIL0kBTL3rYNmelgUqn\nNsS/8IGdp30JNbNTJmXysCjcEyN4kF7CQh/dh8bgJDkSUtSI4L4/HPTjBaqb\nUUrf\r\n=XV4T\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"f8b599aff4ddb386384c0a7609695b65f6422058","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","rimraf":"3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.14","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.14_1613417615548_0.17925348713012346","host":"s3://npm-registry-packages"}},"0.30.0-canary.15":{"name":"@blitzjs/file-pipeline","version":"0.30.0-canary.15","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0-canary.15","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"9b9811b202098c2f50fd70aa60ed5864e34deb6a","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0-canary.15.tgz","fileCount":29,"integrity":"sha512-W32vnvFbIlloa2ojlcE3BYcHmSANIdOZ2z+JMAyRK/20EeYcI73d41x/zz5FE+pKwZHxYH2ibDuFHuAHbXPaOw==","signatures":[{"sig":"MEUCIHL6xegTNYBz9A81canWXqOW5sNM4acwu4koGuBfdKLeAiEAqDSVImv6/3lNk2Px6z+CVKDfu22IjWCJ87ps+368QJk=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415055,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgLIyvCRA9TVsSAnZWagAAklgQAIHkJSCtfvzFOX4vy7KV\nwqoB8mpz/5LS3MEy6zAgY4kS8bYU0c9LiKgs0qTv5WKyLpDvK6E+Z0gMT4ox\nmZ1lQl8fupf3NLlE/akG+uM8DZ+/km7ALUW96DiaMoSuOfN3fABNhWKxFpLC\nMWGB9TJ1IasCnh7znL9JgHjKNJFL6aJ+QDZUbo7p4zQK9OfUPJw8i6MuoIEG\noQaQKLxMHl/eibsSTXHWQiVhsO+xRI/lSxXL3nn2kJ1nbmfZDimSl8eFko40\nHAxHK+oVfHck+7mHDv6H8x2Of/YC1I1Y3rRRKA/U5BQOSqakPbnRpVrjR2r8\nkJPJ457Uds47ySKIjtEqK076Og9uSDz8J/ry6pT4huV1pHgbBPFeMJMs+XV9\nlnEDOlAvqEx3YeiqqQHNrRzJDHiOQM801eNPAjlwSdvW0EoqPFXpJaT/b2Pb\nTp4CEnjyQyuj9AYV/XdJVmJTMWzVJMSlcoeMD8pH2QjW5sT7my29iWBPYfBK\nULcaApyeTBN+Wo8gvWJIcYHYeYPwZCUlWWErO7StxU5eo2rYei7u0lm142NA\ns8pp2fsvbJX8cC63oTKCQx2Xyv8/o4huYQgj3QIQbyghnshySWpqxZgcEDua\nAfLbefoKxTkLb6zEQWEYJEBt2z5tL9npVEJX6DCcJsqQgy+mvjxbVlj/T2Nc\nn/di\r\n=bi1D\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"f0d880a4f2da291104d731e1186171d4758358ab","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","rimraf":"3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0-canary.15","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0-canary.15_1613532334865_0.8353423364616803","host":"s3://npm-registry-packages"}},"0.30.0":{"name":"@blitzjs/file-pipeline","version":"0.30.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"07d7b5a66f075d729a2f4c8a1249d63813b1bfa7","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.0.tgz","fileCount":29,"integrity":"sha512-qdUs/UCZyvFTs143ygtG5gZFQFwAb984OcRhE2YxMdjfgHHLzy8ZcB57GgKN0R+hrMKOtBvcqng9eJox8ZPW3A==","signatures":[{"sig":"MEYCIQCyyXMn+ezn3G24OF3+2r3TRJh6oT2i7rKUzeBdHegRfQIhAN/ltJHumImhSWW35jIuGvUh6WwUaUPN/CDKWPJN7t12","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":415035,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgLR8QCRA9TVsSAnZWagAAZ7wQAJBdPAUW94FQ/J4+vD+G\nYHSD2zvlsRk0pGj2cY/SEnqMkByyrNam///xlHaAAKEJL3aSkcBLh1OHyALq\n/TGv0jQSfudtg8XXKRr21vPmJxdXTUNQZuyVqFMhzMpQnCVdbeCEfueYD1+p\nmktXjbW5+1/DMKFxmuN5WQAhV++6SYehGSuyga5IhhW0cjJ+89lFdEKXClwN\nsabeO5n7YBj3mHl0h4Hzb1EHo4Gn/H0a/ck48s09u3FmK/GZiVF+JC2a1POo\nWqExq/pqkeAb0DShOixvk+xaLf68OYTWamxx+ArRJE5h/oxfn4pY0wQT65WF\nsWTztc7d+AIgYit/31p6JxaMTw4JM1aue8EPZhwveSFRCYH+EXZ2Cn9Z6INO\nBrci4kepBWEBKPGcayx4dEP9kznvYi407GMYsYnZy30PKvQJApQNCfbmw+/l\nN/Jb46TDohyXJqYlG6ouZT+/F1vp92Dm1W8uQf8NQcbZI9gCp4Vilzk6Wymk\nFlX8DMz2d6n1xm5raNrGmn2AgjxKzbQycAd7XRJ3xRqVATPAn834OvEldEjw\nD82wLpRjsnf1W/5cmRJmvovM+B01cssmhcM2ITS2q0scSHERNHXSU0Bp/f/d\neOI0KF0nRFxtAOyOjn7B2MyHZi/Gwk6phAKn7cHNhjyEdwNKipmYDS6bK1XU\n+QIR\r\n=xs/G\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"0a3c975b98e653871c8853f11767fcb4a2012f7c","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","rimraf":"3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.0_1613569807810_0.4510593248717516","host":"s3://npm-registry-packages"}},"0.30.1":{"name":"@blitzjs/file-pipeline","version":"0.30.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.1","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"70be789726f177d804997358f760639ddb50d90d","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.1.tgz","fileCount":29,"integrity":"sha512-CrBMAkAaCkoyg+b2HSwPz9rzCHbh6Hw7haxiG5gyEFyTg31CmD4Ll/DdHRApZYTFLD7935G45xcG1CEw3Z7Xtg==","signatures":[{"sig":"MEYCIQCE+QCch/U45gzF0y2Z/lTTaFeGse8kxHPF3E6tK1OeEwIhAKCFkpyO0ZVkZtJ8dB6I3JGS3RFy6g0JjD/a1PLSSCiT","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":417386,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgLx8eCRA9TVsSAnZWagAADbUQAIGxsMsGy/JVIWN+iBRH\nSdFdWVW64G8BaC3lDHuXrQ6XVBENLoGmKQfH4O+uOuI+oFdaj1flC4qI6/Ga\niiTUO88l0jLj31aUm+hzR5+/KS14cDUsmDtPdvD71zrSZy4c9SpnwH9RL6Mz\nrNBlZoQFs9sfT/uHzJMgHss+XhUpx9cbAcbTpgE+PQMVuQExiF7H647J2+4v\nFR0D8ZX13oB31wPkx6QYP8z6g4sBjYlFP1Oj1mLlcukCRK8kLVdaL0EToes2\n/qYkBAWHRClxuek5DOC4tYJJ2CF/ez5HMX2mYHr1hZ3EAa/OY/yRxZoec4bM\nDGw+5vNU1EHLmL6zIYPZ5SFDdK+69YA8cpuB/2FNC/MZTV0EUHaGC7N9PN5r\ncS5lgSgG+xNwp99p5775TCNxShQy2c0IHk3beGWBiAi7HHUX354kJRJncnYu\n1+dL7NurvkkG+ZEsos/WpvNtRxqxemtBuh9SnpoIzQTmzycoQmnDyPTQ3S8M\n4cXPRVOFkzg6sM+bDcA5n6SZelt4CoZ9wu2o2wY5k/FSO5YaXwLOfOz4kkR8\nhqQ6ziJr+O+dM1xFTljzfjSh9ZiOzvSP0Cl29kFoRQPMk1r3dzQQDWkTFJ6b\nSiEJtKX+ONALyM6+GEXTIDeQlfzxvGeolTI8fDIykQ7UbxY3U/dv15S9FDFQ\nR7tr\r\n=l92H\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"2c4b803e7b3703f6a1fac314c93dbe69e9d34e32","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","rimraf":"3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.1_1613700894436_0.9973016189348616","host":"s3://npm-registry-packages"}},"0.30.2":{"name":"@blitzjs/file-pipeline","version":"0.30.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.2","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"bf0f475f60e5d13df07a6623e3e8d0cccadd81a6","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.2.tgz","fileCount":29,"integrity":"sha512-nsr+c0tIrt9lN5ekyXb72l3DOvdb5smKg5uQnoIplSlPTSgL6Nwm1T4LYEz2nY61tIiYqCu1pXC/OAd6Ik5Amg==","signatures":[{"sig":"MEUCIAmC9Dqum8IoBPiRdDahoy6d+uFo/YSGRWZkgEzpv/u9AiEA6ufQqhvvaAIe0N+ocuXEQCwxAzM3q3POLi+dsDJwnTY=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":417386,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgMXORCRA9TVsSAnZWagAAnzAQAJbNIT1dbmypndXaAiYf\n3Dk8Dxd/GidxUZIUMy4Qu5lZkhK5A6bAhLL99P41lAyROFZawcPi+UBslC02\nLwpFEV+S0iiq3FFXL7no+yvzQj2glGMphWa2vyFcLEYVzhftIQOZSlrWbWSd\n7TisBYsmiAqiv2DlbyJo7cQGj2kOGDtKhcOX6OTDGq7m4R7CtKYftzKboHxA\n7stm2wz25ThlIyanSz2aP2lqrDJ66cU75xOBMw6RNtIvJ5tSVP48GI8QwEzp\n7ElnyITibRAsluhKPEChYy1NNWK9KBAglH0eYqVhceUwuiuBJtOuDEsC41t3\n+52ARqeB3hAJyjTX2g3CGVrTD+sZyh2SK4t/GZfp/dB5AbkhnAcDL8Q/GpMU\nbE6w4plBbHtZ+k86Caf7jwsKhDtaOsE0Cs8kSMAE+FZpHCvdvACD2b6nCT76\nld2d5cC1klkMadPWPIr0g/hQryskkPQs8VdO4Q3z5eX6Edh7Wsd8mc1pqBQC\nvdaaCiFNvKI+y21wHz0VzU62LGGm7hfpeeJ6Jbdd/Mxau/8NA5BAfhpBV/97\nE19ccl0/7nH4b84cVYDSuA3l40nVc7U5EqFUy7Zxx5+zb5puPKyhCr4iAmXM\nfgQ2r7bqN/c7+2TZ1o1OzA74svalwTpZ/0XpjpbNXpH6pBVUPguyMi1V8HnV\nJ1o5\r\n=kjNz\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"d39b461ddd7f61dc71ecb719a0178d801d045deb","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","rimraf":"3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.2_1613853584711_0.6143433197689672","host":"s3://npm-registry-packages"}},"0.30.3":{"name":"@blitzjs/file-pipeline","version":"0.30.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.3","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"52c1eb5a24d1de77d129dd951203efd01c301003","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.3.tgz","fileCount":29,"integrity":"sha512-a6iojJEQRwc0Tsk+1LIk4HbF8OHgJPcteiGN3f6K2Q6FCha20yRBtYUsBdkl5xKcUpNMnf0pZPG0Q4XGEuMXCw==","signatures":[{"sig":"MEQCIH6A3fUIUSl4LngLXdPMtBYc2XDubc8Fvx7RbORvuRqhAiAk8kHanF+tZ1bg/nv4/44jtQnnAsCk5jMTE3K9A32Tgg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":417386,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgNXcdCRA9TVsSAnZWagAAZg8QAIYTdTruwpG1olaqSIke\n6xC9ozhgjGt4Elsy16ZR6sF7TubfdGU5ZQMg6nNihPKa3svM7jPjQSE/6i30\nctCFhlhLK5j8Witm1jvBYe+hQDuRtIrk9rntUhuLVV8V+J/dwtD5Ao0Juifu\nMFPJF6CMwjHLEtRCVlTkPaBaFQ5BUmi5s9zYUJCVbsLE7OUGBag1nLDvEDcG\nX9R282SOGTlp/UtxERZs6x8sQmqcS+LDdeGZaiR2YwzS+k7pKfrdLHapG5sz\n7AKjL7NavbHhaOv5VdcAhLCBmq36Xoyp7K49CEy9sKFsypQHheC/sdyBrxE/\nJHwPikzxE9YOb4eWJKaUPn7j38GpPgzg6forMDdLO1CryXVWZ4Vqbd0dn2A+\nlKBsc2kjowfgxgLdB+1zb9McMjEGzMvBVPWeqTDBuAV4mO25Tf05Zem7y8gR\nhF6P0kx/5XsYmp6Zov32Qk0jKO7S0LcInSnuHqpZJt21gENIOMqQpJl+xTJx\nkZnb0AE8BR4vRU9+94QL+TEayqN753hcQhZTkdCMXbdu4hM7dRzgLUU/3O+A\nQo4TyrK2fMxpggmGWnuPU57IXQLyp3S2aHb7Wcn/m6Kiinr6DyJbHWjyb14g\n4nBw41eCqfizN88R+ryHamtnKv2+nL+jDRVz8GdMtM6+i0d1nMaRKsB8Zw7Q\nXbQT\r\n=hCLb\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"11fc19fee22be7328e2e13c2d29089b0489fb941","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","rimraf":"3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.3","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.3_1614116637200_0.3353670232172976","host":"s3://npm-registry-packages"}},"0.30.4":{"name":"@blitzjs/file-pipeline","version":"0.30.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.4","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"fa2c01db1903380f3f0eaa1cc31230e14fdda527","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.4.tgz","fileCount":29,"integrity":"sha512-i3s7qdatRa3qxsTFNxJztbBJ2IhRP1A0UFtBDolWWoGPuazsmJ7Q6gmqQy2qFBGZ52V1kdzfq1car5PCJkbBWg==","signatures":[{"sig":"MEYCIQD18ReLWDBwWKF85F7XbiCvjXANA0xN+/HaEWHfQUnyrgIhAIsCGTrT/UhCSrfZKk66rY2CNK4DQdmrRL93UKo3FiMN","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":417386,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgNrh4CRA9TVsSAnZWagAA6C8P/Rknh7uD/+N9txZr2Dg4\ncOANASYGdppVFE9fvK9ROfvFxda/k8I4eOz/4EPZ05s9KiZiQRL5dQ8v/VS6\not9Edn88UM6ANIiTliM3+XjbOygm3H9WJSMsh+d8M4pt6K3JFrXeEGTA1uoN\npsSU+WKtkYqNm4/ET4QxEsaYQhk+kYzDm2fyRYLuyrtOVNTh5c/didK1/JY7\ndOJVCYpbJgUa4XJm4V79+uoMdfqXZD3nHzQ5sAwp6Axo7iCumCCftet8TUrh\nbZVpmV/jgpPSoLB/jWFCK8zSRbxKBU+GtbqY2JgfOaLy7ERf8BBSPwQiqRYJ\nJa+x49fW7Mvnq/heXDrvzS9isIL1mWr9CiUvhInYfkVkSWhI5Pn98i4OKt2R\nyDXslZwCVqKvOMPVSqRDMthyGs75W50xkF8XeBfLOrcyC7NU14pndw2cnCtj\nJu95P8XMIeooVg4pFDDp9KeFb+sXnX5ltZm+UiDJPYPdYj4Wb4DkEhHjjX0b\nltoIZdJAd13Wxe22GoV7R+eqcjvcICrdlOmWQWqCRv6C7foUi+3IqqpP5BAu\nRDHo1g9BhjKmaoVa16/4TerEZaOsxVGU/AMICL0VXIIxuQA+vGJyPClOekfM\nu1wWsAI42rpgo79GoipzxYIx774+h+uTyLbP7YoHcecWgAALoL/aSoU+DLxG\n7EL4\r\n=uBiJ\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"47438c8ccff603fb63491b5fa8dbcc59a3580163","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","rimraf":"3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.4","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.4_1614198903895_0.21159063208491347","host":"s3://npm-registry-packages"}},"0.30.5":{"name":"@blitzjs/file-pipeline","version":"0.30.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.5","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"5bd34e20b46c4361682f9880bb90bba4cd25ca94","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.5.tgz","fileCount":29,"integrity":"sha512-kx0kNjjbzTmI2hAWHGT/BqQHPvOgwEw6qlSRq4psQj66sai1ElsrwBo4iswnxFELP3j9ca8Oh5k+3Mv3QK5vVA==","signatures":[{"sig":"MEYCIQCetE4p6kof2suzZyttMPrLMbbTEuvuVYg/bn0nCfNukQIhALcQHOhxZli93wHu7rKMXNo9NxVuRYABO5RBmPf6dvpl","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":417386,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgN8/tCRA9TVsSAnZWagAALGQQAIJ2jOgVZKSQHAxiupqD\nOK6TkFR0BgGrc9tShpPBnEvsoyVlfzKEWk92JWofh7fHoJGuMlv/qAgbi7L2\n99HLZl1ykexIkHbKYXC1ogp0Nr0Iignl+Dv8JgxA1iVxb5IYhy4IqIw0LUUo\nFM8C+gyB7JKebjASIxD0vY1jandirfgv8kDMZOWRuJs6Ocxk3Pwwc6kU9LaU\nKcqLGE5MJJ/Uja9me6JPfm5uHpPGDUs7GNU/WmKCQHlMHnE+WR+EcrwKMfXo\n08uUw/9TiEkXyUUppY1knsUH9bSggz+8JBw+y4ba34mz/oydf5wjZF1EXFHz\nfsFZ4CvZTyINZuabjr29oJGoNP/gN8rtLnczhIZRocyMszoOy5zSUM9hjK9J\nRGd7AfYOo7b1/PLLS5hrb5rM+dHkwYfzTKna41DtLV70+g9Brh4lc7MWVjsh\nQtXqq1WldG8Gw2ikV7Zk1wg9XnAqtWprtHeooN4EvF/hoS9A7SLVDgDjPBLo\noRCktjwAJWZxO7eBKi1kNi8oCInm4yQsHoIc9MXKrJfT6CgwNtCsIaNOlP7Q\n8rQLQFB9bkZznxSD6mfc1iWoPdRI1y23fX61UB45YflngZz+GfInaaQZZBba\nk9JpNvCAoRAKMx1o/zOsZTPRCDATUf/7nGhotjrE3iKMprqUNwAy7K2ScbpT\nUfGi\r\n=c/aJ\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"14ac3d9f3fbaf7635c66a0971642145006bf2eab","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","rimraf":"3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.5","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.5_1614270445517_0.30152522105122115","host":"s3://npm-registry-packages"}},"0.30.6":{"name":"@blitzjs/file-pipeline","version":"0.30.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.6","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6da73bded87d06a286da402f61af57c77a2461e8","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.6.tgz","fileCount":29,"integrity":"sha512-1eM6x6FKNTbJ0RiLM/VQvnDoGAGvMT8XtxymrgJfemCAevIysjmShgW56SzNvYnbovYfK9qeowGaMaHdnobzdQ==","signatures":[{"sig":"MEUCIQDBKrOSzu2YBDrE8wckftzpJ4bilUnIg61gN9iRMJKltQIgJ+FeZ4mP3oWS8q4zqd2pSRwSj6WvoVB+emD6fzP+J3E=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":417386,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgOsTMCRA9TVsSAnZWagAAIqwQAIRKehyuFdsKCTfegmDX\nsX3Misg68OhClTa4BbyRu+J4ouW04kHkqopi0Iy59m3tSQRGieONaw9z/qcj\nxtShOiM6VnB6kEUA95mV4JYbrtD4A3QJ7IPh8jrHbS9XqYu7oPrO8XbX1hSo\nhkZ5w6VnG/pE3IWX9xHwoEYl/Q1815ZW8TKPqTqKjc7YrU3kpeLa3U6fGcM9\n22AG2vnNSyf3qnOkgfgosk76DmXCIJ6VKVu4SZUcwxD6GBdBp8ldFn0XVvQ3\ncxXpjEKb1VrYzJgoKi03cfXsT561sl/2HVCs3KpJIktNDHCg6BHbO0VnP9im\nSZCPYPEBKxEfEwhlqt+GtQqevmcpzDCDa5IpjB6TCn8VUDSpMFQb6kFNCqtM\ncLsjXBB42toVuQm4+wILTD+h3dYaSkc7F0iYgBU125ZEIqnKVKUdGjl+x87P\nnRKyrXTUfh3qIb+n9cqOuRxtIzaeUjkd/LpYStb/Bpfj7fe3bAh3X1f1j3G8\n8EZpQGWuiiplAsar2hDPnaa8WBHRTTJEhunApojIlryqWLv6zztkXFwLspuc\ntWmdpf6dTUgSi/BWS6KoeScpwMz4SKGf9QJfYNZ8wDwJylkIyMoON0BQoVOJ\nw3OKQMs97c8CzeXd9jz8pxR3SwPuogP8QN8kQY6HSMBQiVwlSHIZeFa37rtz\nElZT\r\n=TIl9\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/index.js","types":"dist/packages/file-pipeline/src/index.d.ts","module":"dist/file-pipeline.esm.js","gitHead":"cf6a906fc14c2a6d37042f785e5ab4c79969bbdd","scripts":{"dev":"tsdx watch --verbose","test":"tsdx test","build":"tsdx build","predev":"yarn wait:display","test:watch":"tsdx test --watch","wait:display":"wait-on ../display/dist/packages/display/src/index.d.ts"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","rimraf":"3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.6","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.6_1614464204399_0.7090017022180668","host":"s3://npm-registry-packages"}},"0.30.6-danger.8c211d9b.10":{"name":"@blitzjs/file-pipeline","version":"0.30.6-danger.8c211d9b.10","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.6-danger.8c211d9b.10","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"fc5842a79fe1c843fbf0e86a4d811090586e010d","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.6-danger.8c211d9b.10.tgz","fileCount":26,"integrity":"sha512-Nb/ZKtzoi6KXNl3b9BWcDuY9TtMrrEI2tyXKi8T3BYErMfq6btGD1BhXlcZrC36Y1OnxUNyHhNVU6NuIR4B2GQ==","signatures":[{"sig":"MEYCIQD13M9eHN4pOfvymaW/jdalgBqDBBgZZp815PnyPMPMyAIhAJ8TEVv6kcGv5NxCAXBT4EPABCE2kUafglamV8Tyz6uP","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97325,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgOtxsCRA9TVsSAnZWagAAG7YP/2YnuTMXVl103UFbJi9f\nW2rBEePkXqpvoOQiGHR4HprKybZnG4Ffy2MJ/yujtM3fRVFLqCnk7TtytOzP\nGTke1lTRhvCERlYqnFvatdu2cEsz77ba7kjXD4LAago2EVJRM4+cK0AUla8A\n7sCokPSpbYZ0WOJr0R8FlhogY1gGh4d0UCcNwg+neKcvVC2LJ0iVWnbiS+1r\ntU8hQ/VQBiqLWqbqSNVdNcdTdVj+NuE7n0QoYLBJJvLc/VDaYOAwdWsNTVro\nOiOcOuATVHGmG1fZgcphRvft5qFFnDWRJiSCjMEQNTmZ8pGJ1AfW5cxbQ2de\nXulYN9WFV2HqYgElhgQbz6axBSxgMqM4aNKYrcnkkqQDBHkggtftk+a8xi3m\n/dOfF2PpWa/s9lVUWKHEFFhVbEUUXP6p/L3uWKlxye8yoNvb6t38WDiP+xR4\nRD/ioXmyZ0wNbjMIYvQp1WSpZL5211amT96o3KL4rCHmOOxpZ7KtvxMZh0ks\ngtziWI8D+zhCIRZYYrP+nVpDulflyCH6hZd0i0BJlNGNwdQCbRdoFvQC9o+0\nNubJtDsjc0Vg6ri7TnJvkG135QknEoPkz2Cq6PmdAjjDt7m8LPFF1yOp4DZp\nV4n8r2cyWxWbpcmQlXnfqxm5aObT7+ZYBd1N9FvHTeyJB/JUHqSkmZd9DaEl\nepCA\r\n=8OjT\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"8c211d9beba9cfeeee9f62dc99e87ebfbd774e16","scripts":{"test":"jest --color","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"4.17.21","rimraf":"3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","peerDependencies":{"@blitzjs/display":"0.30.6"},"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.6-danger.8c211d9b.10_1614470251765_0.4405085796387167","host":"s3://npm-registry-packages"}},"0.30.7-danger.689b7beb.20":{"name":"@blitzjs/file-pipeline","version":"0.30.7-danger.689b7beb.20","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.7-danger.689b7beb.20","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"2bf38c6673e280e6e2f0ab1b222b276a983ecf71","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.7-danger.689b7beb.20.tgz","fileCount":26,"integrity":"sha512-VdUAVnOzxW9ZepkbKzdGvPXdHrsYQTebdm6P6w9YZAjmuYVikAz4U9OQEinKBpIzqItZD3dPeQjvwkl+0o1uoQ==","signatures":[{"sig":"MEQCIAo9NTBswVGX06MauE8m/aDatqC1WQIeWeKbRhI5xomBAiAgridz5XqHsiKmLnifQg/eqpGrcJcNPDwLy/UQyLBnbg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97406,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgPrFYCRA9TVsSAnZWagAApn8P/06rKoj7iWALQWWH9ei5\nEgZRmE0IK9PH12P5QBaOk3vuZ8kcZ1CYgQy1PJv3t0XPHEDHNhCc3z9SUfTQ\nCUyXk3DZx6M+YuRKbXCQpYMUJDPo280UFDlWHgJZWXDBJP1m8q1pZKPejRxt\noYesRGoEdodOCsZVSXoIOLh2WMFks1RKeA3SO1q2TzhAac+mumW2LwJxpvAP\nYHZfyutkNuo82r7Ifq1FAB5l2tZJTU1CkVffZdhLPRRAg/W7eojRvyIUeSMI\nC8lt+mOjYla5kelJvN4j38i6L4hdEMeQgDy3UCWpEqmN0JnLd+p/dWItoofI\nJYgppjufY+D5BfcNQ60clcEWwi2WsO9YxwDDLebeGA4ADrJbBqwTbfS+IQOW\nEjVVnd5is2cvunuxtoGGMfE4R/XUUOrdsxbyrx5wKqXV+lnuemwoRy0Gov53\nuKVjDhwk5Cch7V7VC4idf5iESw0+mivMq+17R90M2KqPoufeVZxTbd126Y//\n19GKvnRT3EjjEFLZgfLg+qxDIpmKbUVW4Vkqwox7dGwZLWXy9mMsIK3gB32v\n2h/OjSdw2CtMSpSdTacJncs9RHwGrPGbIhVAaFdKfKjNGYFgBGAGF+aZknql\nKiv1B9SRwNiHXmdOwSS8ruRWLOIJDMkZUkOiZ/1u7ZYn+uQP5JYRjgYbJZj/\niiEl\r\n=0kDw\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"689b7beb4380d46a4efec9199a49ba5ba236f1b0","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","devDependencies":{"@blitzjs/display":"0.30.7-danger.689b7beb.20+689b7beb"},"peerDependencies":{"@blitzjs/display":"0.30.6"},"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.7-danger.689b7beb.20_1614721368333_0.07507402319187717","host":"s3://npm-registry-packages"}},"0.30.7-danger.a2f18c44.21":{"name":"@blitzjs/file-pipeline","version":"0.30.7-danger.a2f18c44.21","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.7-danger.a2f18c44.21","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"18fe16613a9491c9c58e3f2a4e938deca15b7590","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.7-danger.a2f18c44.21.tgz","fileCount":26,"integrity":"sha512-xMdlo/ldVCWn8uJB1bwoBKgOPQLj0C0CiMKhrRTqN1CfeqWpqNmTVH6J57e+q0QaVHZOqYVjXD1nBxF1RCpsmQ==","signatures":[{"sig":"MEUCIQCe8B00AwvxV5KTMFxcmWUOe36QKR0hH8CVWjQ0myUhhAIgRA625f1BRkbNOH2h/NanakjiEbeayTW+c3+7PcOu+YA=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97406,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgPrRlCRA9TVsSAnZWagAATiwP/3qDsd9iRnokM9PNBTsN\nZdtj69dabquVU/PjhxR2Arz8sbCtDX+I2gD27vMdabbIcGJO5DmB0SMjMRAu\nSo7a9svh5UyczAyPRiNE2A4R87Rqua+hMEHqjAJVxB47t66QTiE3PO4DUqHm\ngh/4J3nktIuk42JsSc44OTSYUChXNrDQ1YrIrLFAEqIbkUEEHYZ97sSuAIwm\nprnL8f7xgX9Rlcdks2Ke8Yeeqig2ZrNImS+CypvxAUJV8gxdgQ5i01CpMul0\nNwhCuXxS2/xiB6Eh7rXB3fFIXe4xSsqcYOV5r3WWhuipHY1KoS1K0VR7ju6c\nIYLCq9O1+31Y54po/ZsFNzrUsvIxPHeKvc/+M5iM/+8nGzLJrQqDJPohXPk2\n7P5AQAiF7CrreM89TvcCbgTRcbAcffZs8CQrN7XZpDU+4o+jNWYZoBplRfht\nlHXmf9/H0BPWapOnG+/qsh2ns8oREDjVSDtVB8McWclrVSc1hyi6FZrF5JzX\n+Y12+x3ge4pHkHnQAL0O7NC3M6A8k8NJ1k99UE2k4RX2bvW/HFkf6cXXfFlr\nupAJqhrdCxgngXue/kYUa6pFnLqmSBdJA2j9M0Mk6wgFHMFT6IAOqKWBxdbb\nWfiizwQZuOGP0YS9vZeMfqA3J8AYiv1EgYUrT2LlXL+a8388UDAQoD3n3RFs\nY1JS\r\n=+vag\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"a2f18c44e1749ee17f7585f99116b5b647fcd5d6","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","devDependencies":{"@blitzjs/display":"0.30.7-danger.a2f18c44.21+a2f18c44"},"peerDependencies":{"@blitzjs/display":"0.30.6"},"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.7-danger.a2f18c44.21_1614722149239_0.9199079586390362","host":"s3://npm-registry-packages"}},"0.30.7":{"name":"@blitzjs/file-pipeline","version":"0.30.7","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.7","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"a2071c7fa886e0cab99fdc223fefc48040c7f106","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.7.tgz","fileCount":26,"integrity":"sha512-1wslmqqL4ITMaSTX9s1S5L8iXVXsOQGUqotA5Y7GbvdroTsO8bxraAglLJIwDy/UpB2HhrrDT5I/W0KqC0Zemw==","signatures":[{"sig":"MEUCIQCbQ/1eAILotpgZC0l8q2q3cSjVMaYdKJcO9fyRPWQCEQIgISRnXb6zHYLk5iwNBOQWGhP88mIzGUsekHWv/FjJ7mY=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97350,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgQA8dCRA9TVsSAnZWagAAwm0P/ipdtPqrPb6ZGjWRZMNs\nwH+aeZ6PqBzIjpKXsfywXzIQs6a36MexDZJTvJDfyvdJoui2t3IeF7gO1g77\nAk1XGuDmXaQS6eqQ29z6YOuFOJDH0lzD+A0qrWanFGtlMhuy9uZcyVfzmwvW\nmnm01hi1uJ41rRFAnQWAfFAysvxXVtWKwekYH4tyUI8a1eX2XI0zX5ynzhVi\nTjNruaJoI6dBSI1cZF4K7fVrCM72mUIwLiq1JCzE97J1pp92rRtHdrrAo5bd\nVdHF4oZxkVL1CNUFh++ZG/jupoFgeI+g85/zMS+/YWywYGHxz92G/NbPWUS+\nVq9ngdk5KZxs8TEN6DGeEQ3qLW2dTdhLAcZCmCSR9WA2tmMm+nrp/X5YsKHP\npN/fNCbC8Gv/oGMZnnshzHAsiuB7yAP8xoiHrOag29kXnrcj+r+aYWECCpFv\nGslhuRUK9unpJzyyJlO4qOoUSi3QytxyITfvhK2vzgHFaWLA8twOBQMX0T3U\nzpVuwx7RQNW8X6D1VokjdURQiOoahNJmFLhClazRFum02xBF7BvVvRRJUuSS\nIkGyXfttaif79lZsOVA65xp+NIWzJkMxTYXi6158POAtRuUTUe/kdrzQBlxi\nhEQW4JuZUy08Kz4cAGB6bJCV06YmD5iJwX1q/P3cUmPvSEsyQeKbShz4thzd\nIcrc\r\n=Xgt6\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"2178f11210cee5fa965f446f1b1dccd2b422eb86","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"devDependencies":{"@blitzjs/display":"0.30.7"},"peerDependencies":{"@blitzjs/display":"0.30.6"},"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.7_1614810908576_0.632919747193601","host":"s3://npm-registry-packages"}},"0.30.7-danger.c13a2382.37":{"name":"@blitzjs/file-pipeline","version":"0.30.7-danger.c13a2382.37","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.7-danger.c13a2382.37","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c6003dd7b6094586485ef8656eb5e09f0880a97e","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.7-danger.c13a2382.37.tgz","fileCount":26,"integrity":"sha512-eqEmfsNZcevpiuc+oa2x0vUbwvJrE2O8VK/ShTfMUtUDy9cm7xEb+T7v7gyNn39xNKZHHlwNB2do+aIpePP9Xg==","signatures":[{"sig":"MEYCIQD+u09lWMpo5x4/E+uSVo8YQPJ2WrKqMED+Dwbbu44kDwIhAMaRfkWo0Il2qGZQnvJa3Isd027fG4kn+nqdajHpTdIY","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97401,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgQR+aCRA9TVsSAnZWagAAt/sQAKF63cA+wH/LxxPSZgrq\n3s7HXkhhIe9MRMPGcyb81ZQJUNvucgpMH649+/JHj3sjM0tKgulKDF0cvpAP\nCFvLuuJAZdO9QO2u2Mz9t2wC27R3ZJgviGRXi452Ojwya3kbncayFphcUjFg\nAi/23FmhgQkb2i05UYiNRfbT9oVG5ObAht0fpytxMmJLqoMK60mmOQbR7u60\nElaIfj60fgoccYeRHNkbJgYYd7TUSyktHN68YRVsVErYPHuRoEa+ibq8/RiP\nx9lyO1tBFIRfC+xxZqce650rKUJ2aCwLrYCHXcq/xO5pnEpjBcyw9mPjDeRt\nTDTqchsx49zMOhPt13OssmtiJEYjHfBL151xIs3x0UmAvaDZRtY1BaJvn4PT\nzDr32C4amVmWy5yTdqSGnJ9AxnYCu+qTMw47oMB+MiCjBSj6cpQ7ct3A9pIW\ncl6IkLv/3JTFz/9ogVnBXRVJLDja00545ZZRS75vQA13beEUCyC4ds/InO5b\n5BcpABGhgmAgUbckUMCohwW5j0a8cFxfYVd1jnjKw2oTMA0fiQ+2fLEq/VvZ\nIdprsbhbvaZuWQvQz+cp46HHnmVGyquwKFuxEA8Elz+TtGO9KBEIplvAitQW\nuxOPsUjfyosNOacTO7sCMoh4NS29CW32kkXhXCKtn0I4I2OyFIaabbZ3zaRC\ndTRd\r\n=f1Rz\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"c13a2382d9e7a5705da0f974aff77956f2e84be7","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","devDependencies":{"@blitzjs/display":"0.30.7-danger.c13a2382.37+c13a2382"},"peerDependencies":{"@blitzjs/display":"*"},"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.7-danger.c13a2382.37_1614880665612_0.05720449211237488","host":"s3://npm-registry-packages"}},"0.31.0":{"name":"@blitzjs/file-pipeline","version":"0.31.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.31.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"061fe873ac325d88455023f5ae30bb40fdc52820","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.31.0.tgz","fileCount":26,"integrity":"sha512-7y+npQqOdut05AgHo0dSi+ElzQ17jWr7A3Kgwd0PBaNnQ112oUK7JbgPPnZU4aDGAgMSQviwzpOXvj6ORk+5hg==","signatures":[{"sig":"MEUCIQCs2Tp8S2BRvZwf91rc/ee1DlxPa9Q8/d2r2hhyTi2MiwIgHUefn8utgGlM+DPkcrIvBE/i/dIFRayS92zJEXDTxvY=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97345,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgQVPvCRA9TVsSAnZWagAAWWoP/jGmjYdEYy+EQIIOMwTW\nRQt7/472pzPU0341SWCVq7N4D1rgfqkWo7FOnSfW0xUgyJtWsS4FS0XubqSb\n8U2wzVpmX2KmVkL7Sj0OMzE2Ynb/SLNjrcuTdR0b7UXu00lsiGxQgIjVxlk9\n34gZYrJMmgh3RaZZ4Yr/WE+xwe5zxBdnPfe6GPs2wcEyr+TU/vZalSnZqnDy\nejsWzMoXhAw1056UIyTtrVi0FbIkAngznD2V2TeDF4feP4bOp04FevQSNOP6\nhDHP/WmZetx6kX/zSgZ9i/OOW7TvSqrDRm4dWyDyEeAoznJNP5F7mn+UQVAC\nzzG3rhoKqXym4weSaEOfIh1uuVQnDfCm6TJF1KdkqIpIC+PhO/BhNqoRcozb\nLECP+nkmMatATMa9b0RISAKcCC98OJ5jjPSM3vYv2AS0NcMbI4Prwt2ApZo+\nVQOkjwwrx1gfsW8enp76sWeMl5d9EvFEdNmBOjxoJ74LLwkoOWpvQCtwNNeF\n/TZNAcb9DsQikVI2cfI3EZptycK5gwocEJ2e7JxxkpoOX5XhBsXbccf7FSzy\nnuxJHWbdqw4zuUxHwmqVMOrMPtMQHLjGNWthg5CTL5R78VhILTBgZE4Et7nK\naQ9VQtGcEK1HritJEtdrDxH54g+9uZvYcN8KrbfLmCoitXmq0gnd+AUeP5f3\nALsU\r\n=xI7G\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"0f9b19cbef73a226c0c9cc5af36afe776b62fccd","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"devDependencies":{"@blitzjs/display":"0.31.0"},"peerDependencies":{"@blitzjs/display":"*"},"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.31.0_1614894063470_0.8123211869196001","host":"s3://npm-registry-packages"}},"0.31.1-danger.7bc800c2.6":{"name":"@blitzjs/file-pipeline","version":"0.31.1-danger.7bc800c2.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.31.1-danger.7bc800c2.6","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"000f0db0a78e73259c0e68d6653633dcddd06ab7","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.31.1-danger.7bc800c2.6.tgz","fileCount":26,"integrity":"sha512-Xv1a4fFsZxu/HAMRhBroKcLGR4FUuh24l7ImIQkYSq7znLbuZR9r6BoBiAwZqF7bYhrpIPdmLCk48p2T/Jfb3A==","signatures":[{"sig":"MEUCIQCBbLFOLvQStjLAB0ehioyhDMgzqQMiErTBcaSk9aY/WAIgdjBhtTh+YveDbP0ePvYimpJytvhKQ3kPnZ90kcRUVFE=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97399,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgQlqBCRA9TVsSAnZWagAAInoP/RspcDn/VMDkOqxCME1s\n7Lj3x3vRTGglvJPPSIfHSBaQBaHFwuckarYpUS7D8kld3FXJCeFnsniogoz1\nMrpBFR35ZiWJSareagsPaAQ4Er+YwLgDfEhLmfWCDXVL+yjmEmKatzo4uP60\nHmND2/r765IueQEDhuJywxhaaniX+YhjveySEnKX5K1UhGJk+fi7x2KTdwTQ\nWTkAIjiHAq7YMCNA/JiG2lnTSj+CTYjKpHruYiMfR5+M+VpzXw77VQSoNYoC\nIL6jpSf4W5PfmoWQOCMtXV6NngB9kSNqCfpYcjk0jVNPLFcDtG1y5ZkPtJ+S\nmKGnNhteZcz+kmMGeG2tJaIWUeHgRU33kefdvFCXRcM0Z4B4+53ax0VerPBB\nTnbuoXTaBBy0fuW3/ZkJg5tPTvIruSiAxX1uJgYUL0+1Nb7ZiToIZpuO0CSQ\npSlPeDkGVhgdzUwdyTH8qojccknJ0bR4tRa/fncHtbiQ8Ag6V4A5Tuite/sC\nCJ+AIifCpOPwf9Lgbw+giQL8x/N6uWsIbHbEn52BliT1aRrq7SaQM/ZcIswT\n6KnMKIEIDuoiq0XrqvENQ5Hzxz1BC3ga7dtbOfpnVkPYKVwbUs/euM4d/OiH\n/0XJiecaMsH/zEmHmBGzM5Ljtv7Y4sD6wQM2gPEwWG3SC8SoSnOtp8tRt4b2\nSkq+\r\n=pqD+\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"7bc800c28c62fb3fd5eda8e2dd52b6342f2021cf","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","devDependencies":{"@blitzjs/display":"0.31.1-danger.7bc800c2.6+7bc800c2"},"peerDependencies":{"@blitzjs/display":"*"},"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.31.1-danger.7bc800c2.6_1614961281402_0.7964109382807798","host":"s3://npm-registry-packages"}},"0.31.1":{"name":"@blitzjs/file-pipeline","version":"0.31.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.31.1","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"994f6f28a65c619a9c04e41280110da1a0d40640","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.31.1.tgz","fileCount":26,"integrity":"sha512-MuqjqC+GZrkHUyVr9g4JTW/ZOW/4SSqt7wU01JRBYvLS/6gkUMTtiT9BmcHdSZubk2EosZy1ddVsirhL5blDhw==","signatures":[{"sig":"MEQCIDvT1xZ2tgLLLKzlMzAO/8dDA1TB/ETKwB2zp9QjFJscAiA8DW8vMSdL7ESj15Y0A7DOI9oOnFT17mrXt0yD56fVRg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97345,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgQnMeCRA9TVsSAnZWagAAiI0P/ivwyhNx38+gwsTlj216\nb/LU792TBVvqQuj6X+1zzjLxk4io9gUTWn2TZiotzXLvktDOaG2zuy66U3xH\n7qdD+qAE2OWdQpqbxpoAdl1ntuistgzk7mlMnPKv5zGCMUo3mQ5NH35X/PcG\nz14x2285e9RqOJl4S76EiO5qx4Tl+P2o2Qtnby5dH57LwJvkXjyULEXkQual\n0IbEDvYjC+Ey1/Q2HuVuQ5dwGZ/OQhkGdt3q8EfxpOdX5bQjvGdZBqohUkvb\nKQ+hjYDo6up4pCoo4FShYBPl/0vyZe04H34R3jVzjInFWfbTwOVxooj91MwM\nwyJzPNs9ldiMoh2JuKHl+VpzFrBjRrGwwhA8bzgPeattscVl8A8kg4Tmb1VO\nn9gvZlHrVkazd2+w/THnwbYAbbABlXr+iV6W2C3JDFS9NTPORh7ZE+DCOVFX\nBcf20DAP7jnSdKsV9tEP2M7qc9xR9ZYRLOz65DXpNYUarlwzut5lFHNJUpw6\nhJHm++xoTVyKC2YC5ROHkd/XdHmX61gW7Tm66YxS0Sh2DfRUTvACHl1xmuLP\nYptBNg/bVf3reivh4cMfWtLIlWuucQAzBTIK4V53m+IV7ncod5jYde4yr7Bf\nwxlugkcf4cqfelUyNssGgO00XiqYRo+7Z40B0Gf4ZyLc5ZP946YbhDk4P4CG\nYR6n\r\n=Qiu3\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"ca7ad291e69a6e79b6c6630d49fce273bb6a2297","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"devDependencies":{"@blitzjs/display":"0.31.1"},"peerDependencies":{"@blitzjs/display":"*"},"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.31.1_1614967581436_0.40363019385855514","host":"s3://npm-registry-packages"}},"0.31.2-danger.d90edd13.4":{"name":"@blitzjs/file-pipeline","version":"0.31.2-danger.d90edd13.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.31.2-danger.d90edd13.4","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"89941abbbaaeb2bed7e8af4e59eda15b809c7b69","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.31.2-danger.d90edd13.4.tgz","fileCount":26,"integrity":"sha512-UJbclxxz32oHdBtUX8vYYALe5hzHxixW+BnM0Ckhv8wOQWz6YKkm7I0JqUC7gYmk3j+way598h8FzvCzkZkOkw==","signatures":[{"sig":"MEYCIQDL+Cbc+x4bwDRL8opt7R8x1q2MhzraqMaO7YrC4UrVVgIhANT8Vqwp4ojUVNYoUsDMyZJ2Xdomg4ioEoOIy7/DJXSJ","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97399,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgQ+zOCRA9TVsSAnZWagAAvYkP/jAfJDHv8O4k4XlxhGa7\nui0EvvzTog2i9DVrrYuO/yjQUL814oSdbABr5LYY4xN/9lcT1lMppfvoG4bO\n0+LCUaDs5a2K0j/Nvbt84mmV06LjjshYntVBOqWSv2FGAt+o61pzXcon7Ash\nmrVdEjwN+VX2Qc8+B1tvxbhmWHYuetRznFirMFktVxd4UqBuVRS6XE8lRsjL\n46CzECXVWMJLp3mRJ48HLURF1TEWMMEFfCzlfMFRpI+OZYD9d0czQirY6XMK\nv5gqkHFm4EOVlFms9oiBQ/SDYNxrc4RxO52agelAdAdTdimDXkaHuoil08Hx\naGueZEqhC80fBFHN7/9nqa9hSTkM0jOlUI5zmm90Pqiz4kCJF7ToM8wkabH9\nVSDLJmjUF9JvNaUuRc1m8M+QLqTXp6v0+/Z7xM0QH3ix64Ktic/UNB3witVy\n4emsAqeUjFJR2xiNqtuh7UCZnG7aZ1RjsD5YBOG+fSyoJMZesAVrSKZUQxpd\nzWaCOGVrIH/b5Pl3a4NlDACb1ZMPrkhgFq7DqNVkaLIuW9/pnwRnqd0ACheh\n984iSOiq8yNQ4cbUBOQYNuAqc3uDI7Nk96QZSROHbUGUQ+P8XpmA1XRgYX4p\nKoxt/lgYgg9eILTzmyHjaTN06Q60E5CUX3DXNQQ5gBbsZFZ80Q4AVBeWdQLi\nhu5J\r\n=oWhq\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"d90edd1314adef5f2f8eb1db6f44e7b419a1124e","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"skn0tt","email":"simoknott@gmail.com"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v15.11.0+arm64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"15.11.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","devDependencies":{"@blitzjs/display":"0.31.2-danger.d90edd13.4+d90edd13"},"peerDependencies":{"@blitzjs/display":"*"},"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.31.2-danger.d90edd13.4_1615064270011_0.6514626578520255","host":"s3://npm-registry-packages"}},"0.31.2-danger.7a88537f.5":{"name":"@blitzjs/file-pipeline","version":"0.31.2-danger.7a88537f.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.31.2-danger.7a88537f.5","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"2fe978816310d6f25e4959b93ced68df306fbc0f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.31.2-danger.7a88537f.5.tgz","fileCount":26,"integrity":"sha512-an1M39J3nMlljkexphKOOr2oLmB717qNmWGeRZGUBuMoPoPbkjNjERpi0tW0YoQGsDXUJt/d4WCKTcvQiMQMqQ==","signatures":[{"sig":"MEQCID4BS0lmeveuSjfsNT1IeAyW+YqomwdU/YSX6bKAidxXAiAS62WWZ7OHK7rjAyxV5sUNLYVg2BcODNS1TPxkLhEcuA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97399,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgRpdtCRA9TVsSAnZWagAA30UQAIkBUOU4gtF/op/ntMIZ\n5Llv+KITretC+f/NvMOoaoXjWROIphLCaAoMeyrf54J5lDq4qNEpqFipipGF\nChQgD1bd4GPTRWW+UfYb5hxB/pSSAthl9vOFSs2LU7Wx9q20+IJWoxXDbhKu\nxs1Z8K3BT5zHqYxrOoSDr7icbTcqn031gVdYolQY9dRFLUBvE414ip6PhlyF\nN6e3mxfhPef7GGMvx9G5/pUiyA00fmvVWVQVQECsY89fNsrzVQVCp+KV5AjA\nwBN/5zdKvnkXhDIdUgya6IShybr72kN1BBq+mshW7M5AxjzWdGXpX72kzUY3\n5wqETQWgXnynEWKlnD+vXSuDcbblbt1/41s0Q6Li16WnZAYcY6fuR+KSQGU0\nxTrMoiQBvnRC/aNCr1tRalyDEWuxMMRxXt7q/8QkJNVC23H9cMsFJuiE4BMh\nsbu6ZuNS3DMCZEtiLUdhKFjQI5zWFzzXYD0WRUSehBmUPrcxOslhj5kTPz5e\nhklujvSRyIM8DjX5kwxY+fFCjTEAP7F/fW3V7Nwco4vGJdMaajeFDcOkZYNc\ntYCaCGEmEjF/CCBH/OHt62wTVYu4yrH5j/NXLGC2uiwT/TrkcUwhroB6bxC9\nznq9x9QOnh6Z7bhSEAjPJUSwXJY35N0ESQ6wwHvFUXc0uUd3Bw6Umh9OFZ9m\nQN1R\r\n=2Njq\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"7a88537fc530a96f4445780656890ca2a9d143c8","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","devDependencies":{"@blitzjs/display":"0.31.2-danger.7a88537f.5+7a88537f"},"peerDependencies":{"@blitzjs/display":"*"},"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.31.2-danger.7a88537f.5_1615239020934_0.18884942375569302","host":"s3://npm-registry-packages"}},"0.32.0":{"name":"@blitzjs/file-pipeline","version":"0.32.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.32.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"23ad592bb9ab9aa868adc91b5005f34ac792411d","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.32.0.tgz","fileCount":26,"integrity":"sha512-DER8rOSdp+7fQH/3taH26nB0tdyvJ8W2DVIV+0MLqkrHueHUMPUxtYU2wsULJYlB1HhbAW1Ds6IxDJKZWrvs2w==","signatures":[{"sig":"MEYCIQC0RO9iz3XepvvzzGFikA0QmP8Q/kIq2jat3q9RBUFh7wIhAKInPLYsrLVaxdF/75SuobeJW93Nt6wSaOINa+60wImF","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97261,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgR+/pCRA9TVsSAnZWagAAMEIP/2mP2n0+08WsVhbk8wFo\n05k0olUwVZ28WjP5LEw69yWoHrAgRPKG3MSN/1KpwC1DCzNk/blWiN4CMDen\np+1apT13VO8fbA66ZS3CExKJ7JDWRyb06xd0HOHsjY/bINTEgkZOCLZ8rpGy\nwOBUpNnzY34i2vLrDZ5p8ne+bttqjuJuHgaZEyZsmDzepSlCtfsz7yjnQakq\nvbfmQvJwfPjp7Arfx6jjeMf9gzbAxoXSpHKnnGrFTlIcvRYgz9iHtRb+651x\nBueOI8HiiVQ/iNmbFFiLbPDtkpXOqCG4QaQoliMzNhHamQxiveokbqzKosPk\nZ2Pulmp2sZr3kRghUgCyfCcpkQP0sFuu4Amu031x6XzbGAg6PNYiiZ6N2Yey\nOPCEoPa4N5choyl2M3vT3DdThHrVFJYZLtBluDet1StnjshmAI9o5z63lPfj\nTa/1IGJgjAe3kO0rgJAwDV3iqHTOkObfhvsLi7VRZ8yGZSKiTNGQXMSgClxt\nanSWSRUF21zaS2MOKf0zPwIQFlEq7IpNi+DnrwQ5oyWigWZpJCqgOYQT3P1X\nVEbpwHjLJLd6aHiLU3hsJijZD+eps1ockUzUxy021nEVxfIO9rV7m3X+9upC\nKMuhOybR8aIZXsJEfd7gcqi7i3mIkLPSssoFSrh3rhDiSCkFgKEMvXsnnCDz\nDEh6\r\n=Z7HG\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"3b114e0b37688b8d665a5bad56e63f4eb4e0d353","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.32.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.32.0_1615327208485_0.07798456693856948","host":"s3://npm-registry-packages"}},"0.32.1":{"name":"@blitzjs/file-pipeline","version":"0.32.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.32.1","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"492eaeab86226f23b0840e7af33f46dc81bac423","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.32.1.tgz","fileCount":26,"integrity":"sha512-QVwM7B++9MHkef6Hletz7xaTroc/GA3mlKFaVeJRxVrVd2lMYJiiS8pH+gQ3tCMSJZb7gfENBv4UPpu2vjUlBQ==","signatures":[{"sig":"MEYCIQDe/lPLN1Ql1/zuGTJZCi6H0fOCFdiLTTrsc7z5EqubbQIhAI/kIyhWLVzc7efKUC1kqEEuFWpuw7gToqaASDqlkOb2","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97261,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgSQNOCRA9TVsSAnZWagAAXzIP/1+id9o02eanbuqyxt2k\ntJ9Qf7XghHyLnNpeLzL+n4OEbrxzhJdRxwZb+hy2Rhucuf63vGTTshJTf0qh\n0xnE0Lrth83hmWSR8Bv5dReYlPJvzytBe/ccOh1C6aOTFPSYgpPDviVXCWgv\nQqpuY4NMW/xdx3TmhL5pDUUHI//yE467tiwozx0sCCvNOPqNTB6DEbQ225W9\n2oAPqJfJEnFhENUj9rHfIfNax/9F/6jynRCLOTRWYiR6E6g5RWFl9nlYA7jN\nBprZprijFfTJMcEhmGZVd//ErBttADfo8NwGm/YE/+uLz+R40cZi8ClVjDZX\nLFEnZuHqty9j1eq8Vkb79KemX/8Il1cEQgGNVAOzA5N4StFLkA1Q5tqFDdzx\nhAlwFOH9rPyn48m1n9WCha0FI51gwgAEnVtrlO7AmNE+r+6Qn4lEdusTscyj\n5+Oh9FqqXkyAnK1xiQ4PIymWfQjmoRMm9VQma7PiXsFDYqvIUg9Mb+eMMNqF\n3BiJ8KebcW0QBMuJ1mpgQb95Dm8+W36N0nq4QaERu6uddLeGpWcWWnP/Qp19\noja/Bj6SZeP6fbv72mBTQ272tTm2/fp9kD00pcvVrXg1Q1YQX/UUsNZo38+8\nFzOQBX6cngEriYfynBBt9TBw2J7kjAfG/JUQ5QUVMlbPVtAulFA9AgcRbWzO\nzILI\r\n=Izqa\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"3812b907a4e2e9433d4e1b027ba6ff2d6e976b5b","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.32.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.32.1_1615397710099_0.28018735694268804","host":"s3://npm-registry-packages"}},"0.33.0":{"name":"@blitzjs/file-pipeline","version":"0.33.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.33.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6e59b3ffe05db7d73408117a3d6173370b7e420d","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.33.0.tgz","fileCount":26,"integrity":"sha512-dHq5+GQvsYDwfAncrGP7d7bE+Az3YC2NrISgl+3JOV1G5hjrrVUQdcR6gUhKTxes8wGPiSiC/XgmtLoDj/VPRQ==","signatures":[{"sig":"MEUCIBo8BZDMZ/xn9DCF/8OPYL3zRVYi4uXaPB1rhgNftJf3AiEAzqUNjOkYKMAnuOYilKmEne+9QDhMnkUs7oT3l7nl8F4=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97261,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgUla+CRA9TVsSAnZWagAAgkYP/ixeu4faG5KyYfdh+5hS\nE0ZhT0G2Jj09/ocVbDuEGYmJ8AAXOY3TSJNMSBgHSUIKiXavmDcOO6zAn96M\nLt4f0qkf5EG3RJjN95eXxEptv9u9VKWZtY0xnhrd/3oV9EZOpcUDc0NdtFbf\np9z2ANH4mAtNnFtf6yzE6R2/2f4+UWGi6lT3RrhFQp0/Fn9lF/QTenPY2b3q\np5ZQuJ2ECXOKzhUsSahHmF9fnT64YD+F9jOufU1000eQKl3B95cZOnQsVGjn\nB85IiJaToUM3bvHzZC7Xpqm4qgqQJWBjpWFzkO/ODCS72x6EUODxI1rqr1jH\nM7+ZLpmzHP2qcvAuhRB9lErT5s80hdr04BEocfek8iY2aaqE5wdDmmK60gIF\nUbP/LznQ/jQaiWYbjK9opKp2LUskwi2tE2MEoLDssbi87++FI+NE5HVfupyJ\nv+BH+MTzuNJfZ31CFx1ilhj9IwQD/IsC4vUnO7xjUn2N9MTh4jVm/fw7z5Pi\nUt3YCkegyEkTk1XbpnIEgK9B8iobbHQOkdXVLB8kUaqwTbUfavnHaNrN1LdI\napJwguLBg+8YOn3cgOeLvYne+MGTft2imJmb7Q4eRf6p9oif0QPoPwM/OOzS\niSqXg2gMQ78+m5yXq9/KtBnN9z2EDSeNL8DA/xzdg3qy4wW3hRqbHollzwd5\nTu9t\r\n=63ws\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"d299d5036467a393c992d877630bf2771741e99b","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.33.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.33.0_1616008893565_0.16458327838720388","host":"s3://npm-registry-packages"}},"0.33.1":{"name":"@blitzjs/file-pipeline","version":"0.33.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.33.1","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"73cf82f1337756b608d9b6e72749a4f09eca4ed8","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.33.1.tgz","fileCount":26,"integrity":"sha512-E0OWzngW/Y2booHR6DqJNbea2FJYSJAL5KjkS8veYgr1d2NIB2M8gx29B9HdG+ki4IGA0o9S7emCeK0Hl46uog==","signatures":[{"sig":"MEUCIQCdNY54y1utH30gehxEk0dIY+VRpg0pkjK7jZeretxx9gIgPa3CBqpwpV9OhL80gO8qdAtmaxdv67+noXEBfwHTKqE=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97261,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgX0xaCRA9TVsSAnZWagAABLkP/3wrVicaxG/+k0iLCGm5\n6Fjfi8fugCmahXeXRrKRBra1iAG3177+e+nKzPu5c4tyckgRzvl1zc6Reos1\nE7ubTJvZscC6r7cHa8VP8iYUqFOnUKsaXj3iaHW5XqNoatJjMJ2kH1uoy17p\nTIpKcwUOSGJ1XuQYcpe8C19nX89YWcH5MUXbLCQAs+9YsSnn1CZOWsy8H8oY\neUjJN3WObX9uAJsHDOJeDUEt+n+1AGyAbAKJuYxNxdZaJC711FHq39MgpVha\n9X+PpMqhKYmj/RXA9Y4on4PsmmREkzQ5fzF4Upg4Ak6b0LFj2AS7hoQbFE4/\nHitChywxVym79WWd/OYbebjXGMFU7aslU1e3REGemJQ6OL/xL0KxIx3mPe2J\n5SBIUs0osu11aw7dRfiBiR73GKn9sPz6zOV9dPeasNiTMkJOwwhZySJt12KK\nV9HHJykmVZp+eqoa0z6oq1HnEQ6IOG+KQhwHx7SUwZ0zF6ent/woC5o0ClsI\nhwTGk0+V6EskcmzTkN1nT5yt3kFZt0R+0hTDYpMdKPmzt8J+MSt0264o6sVm\nuxpgbwS/fzfxSKhwgksoy8SgA8TbSKX/ECvl26a6wj6MDh7wkArUZVCEFH74\nWbAlcfnsXMuUewLb1FnyaHPJSKEZTEgJxjuOyNaAZro6bO8mOIQFAd4jKFVV\nmZzs\r\n=Cmgo\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"9a13b4e361364ddfa63735bd9ef5f4da56e40c0e","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.33.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.33.1_1616858201602_0.8616828501467932","host":"s3://npm-registry-packages"}},"0.30.7-danger.d5caf8725.39":{"name":"@blitzjs/file-pipeline","version":"0.30.7-danger.d5caf8725.39","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.30.7-danger.d5caf8725.39","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"ce7cbbcca353a532963cb61b1069c6852e0514ac","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.30.7-danger.d5caf8725.39.tgz","fileCount":26,"integrity":"sha512-bJ/V2/BT4TMdMDc8fvbyQrKRML2+5JhlfMQHvOTQVJAFedCaRYb/ShthP1HKUYeun3AqT6UUyggNqoC61Hz5gg==","signatures":[{"sig":"MEUCIGeN5IbCo2xgPCUGKVqDC/Pve+uanD1ya3ZObJlx05ShAiEAmiFL5VVY7iVK+n0wJxsSPmbVhsI+fD0m50rgWFSc9XQ=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100349,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgZz3CCRA9TVsSAnZWagAAmMUQAIL8LgwY7TnI5ygAA7Cn\noYOf2cwnXo4W4aTgIC8oVPq0veLBrPIjfrbiYEs80DNwR+IlFDVG4lGOfEl0\nMzJSUQjY28qpN6dxrCEesUZYaEYLmDDU8qdMYEjBBTQk4sUmjWcuipjMomLZ\nipe7wYOZuXzkyCYymMTvTJgGdGwahYTFLzQZig/ZQ9dbzdqB5OeZmGRlDvwo\n7l5mtD3T8NHiA8/IMm5VP0sB9XHVxo0cW+RtFEOy5WUQv0VA6Ys1ACxwruPk\n6gyhYDG11JInpb6MUVo6OD0UfLLLiyJ5uWcKv2ZPg2AahpHVzzpdhTUTf2e/\nh7ixDNy4Ov5OxQs6Mi+SXIe0BMs8iKBk1ocPCYUbfGXgOH6GtUqfr8U9srln\nzT/Vgq00SOkLkJJtxE/nQVlIbA3v5h8vj5/O8UTZN04T2s9zcPzWuN7vRpZZ\n79Qp8rNJ3KsSjgtoqH5ZzJd7O4Q2CYiu6CkkexFbQXqr+xNABhg9G3WbY9uL\nUsbSEfD9qkz0CFXRRnLRAdrpKLZMrnuuNGASeGI22eqBrgqUQt39vUN9XT2D\nZ4ugSd0H9cgjfYH/YRcRyUwSIotamrGMTtmA0Asmgb8EhCl7H3XvIEoBqOVN\n+1w0mYObEe1l9xZElQk1BzeQSVOf4nyURDALC7bfHsWVVP5XP8WCSlW90Fhw\n34pN\r\n=lhVy\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"d5caf8725d36c40f30f9c8017d2858a4f756f0d3","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.30.7-danger.d5caf8725.39+d5caf8725","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.30.7-danger.d5caf8725.39_1617378753852_0.27475011396645943","host":"s3://npm-registry-packages"}},"0.33.2-danger.776e20c3f.37":{"name":"@blitzjs/file-pipeline","version":"0.33.2-danger.776e20c3f.37","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.33.2-danger.776e20c3f.37","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"1e00e0679deff62ad37beb71869ef3a2af2bdc92","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.33.2-danger.776e20c3f.37.tgz","fileCount":26,"integrity":"sha512-ZvGAxW2mwxyiLPfU+VB2RclT7vPV1cuMq9WZ1DHhHt5f2Q+9cVXisqEhzAvgE4OwD9oV8FmPUMHBTuzZf3NVzA==","signatures":[{"sig":"MEQCIF2cmkCJxITqM0QTpEHsZYGKkYvIT+/upljAnIq4xrzaAiAI4qE3CmYlvCuop5xlvglH4h6uHHDGNRa33v4nnFZXnw==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97321,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJga3WrCRA9TVsSAnZWagAAyK4P/2tX//JkS0tpQOjC+xak\neI9uWyJGHXjL0k8xK9lEHoAKYugD2H0eloDssTbhcbrIgbGGFWl0FOHp0C4Y\nnsb2p3onhQ7IakMIoMJMhQmtXgaUjCYEUwawiSBOkPqolfh0UXOTzvRhjtas\nbCsNIuSdNQ6k7MHyD2tsoDmHzOh7kRUYEuEUvdvUwTeIqIFx97AwiQTKmM8g\nNUEiMUcb3WN13wfhiux0M6mOz1irf59IARGBBHxB9YJpDKuTO7ipbTTmaNiK\n5QCRonBF5uDysRVTAhtRzW3C+B03GNyHPIPAcLAtP2hdCygu+a3GIsKO7yIr\nSIVQWD63eNvdZhhT9LnO/PtSHG/dGt1n/UkKj+0HOhNLmfuMW5OXrcnFvxs4\ngRn8kLtxgPRFs1tBlbYlyGtoYuZpJ3NBTSFoDqly65wonUOp47mNAU4C4WjF\n0zKNIl3YZTt7UJ1ThsZFH2JyyELGR4wl4zmdCNlAUnlrhOM6wa5nfwXB2PAS\nh1edK3yzN8SfjzXzp7SyHwT7ZeDZtfRw0wOggStC+3mkMFZ1tEOvghVyR6U6\nxf9vche4hmzfRZbDsDrFISXWnBQUl3QOvenEQMvb/L5UUGewkOTAYh+tGTW3\niuxOyQB1muSkaVJpZatnT2xOo08e99m+AYQpMILNpyREHsKux6Uze0/bxh8x\nqkzE\r\n=/uth\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"776e20c3f8f832a3391c2a8979d44d71997b42f8","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.33.2-danger.776e20c3f.37+776e20c3f","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.33.2-danger.776e20c3f.37_1617655211043_0.769599791844434","host":"s3://npm-registry-packages"}},"0.33.2-danger.192318af2.43":{"name":"@blitzjs/file-pipeline","version":"0.33.2-danger.192318af2.43","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.33.2-danger.192318af2.43","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"27276893d622d601afe61e30029d63ca3ec7a5b0","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.33.2-danger.192318af2.43.tgz","fileCount":26,"integrity":"sha512-iaXY5ADgerqDwzapQtTgbnxgWUD7CxyNlrxq1RKkXfo2q5FzFtbTu9E4l2NV+V0w0ZdrMDmUvWUbHD8duEk1iw==","signatures":[{"sig":"MEUCIQDexwHXvXuxizPQ3eqtC9GYYds4U2ykpU792Bm2ol1UTwIgQAo/G1WFKqxLzV0tJZUYPY+FPvegtVA/kPpuJVBJXww=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97321,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJga4FSCRA9TVsSAnZWagAAJBMQAIcdmRP1WlBjzHq8cG/D\n/QCQ5wvCDJiutKCM5TIlQ76jAP2qvzmMrQdD7eZhsB64vu/R0mEwcOJXXeXs\ng1Z0nWB73xrmxMS6iAsqr9/81vj4ENSi9Sm45HIuQpM6EF4TbVwCBisfblDr\nzeqSHhL0ONknB4GiopW0QZDIakq8ERa4EjuQltDPT2AF6uLMBt+x8RlB4Iib\nseIGqj8EcrxXJlFUl83XC/fHw5zfwnPLENtpJOmV+DcISpzEL31mOpZfBMmh\nkVRpYqu9NMXmkwUu5axspUERvXXsY5GHdJGKg9Y7PPbHvy2+MwGlmPndUR+C\nhbgMdeXEWJmBIW/eG8I8sXQEgijnfIM15feSLVmTbBkAsFvIeaFZQL661iNe\nuZKt1Crim4uuqR+rwCS9lwj/qc/tmUnutboXGtpzWR5HK5OEcI1xNYMSIdFu\nSLQoQBcbDgiMuCt+oqijjZxT5oI9SUBkY2qfqnHC1FdtYkXvB3rZeOffn1CO\ni+2EXMg65aoRENiqDoLr1qINls/KApcfo4IS3IisOAjLFmCY6ZJfvkBuEJhd\nQw+2exlMFbh0jOGQQvJjZjjOuYfIoiBiA3zP5VbD5lFY2qhclP6ycGYj+3hZ\njIS0PMyPthSEDKqhQuYKv7xHLbKb5shXONuCtEef9K8UHyu2d/Z4p9ACPi1y\nk7CM\r\n=v9Y1\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"192318af2d3ece1a2d40c4e047d5beeebc7f5a81","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.33.2-danger.192318af2.43+192318af2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.33.2-danger.192318af2.43_1617658193917_0.9693663377021582","host":"s3://npm-registry-packages"}},"0.34.0-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.34.0-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-canary.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"80a1ca0a1021b8d8c1e59c9868296ab35a7fad7c","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-canary.0.tgz","fileCount":26,"integrity":"sha512-I0IpeoGqeIBFW4riUvhQdt5gRwTKBxaHFlBpn+CtLRXBDa+vcgOYDFk3d9JBB8VYmzhxHsLKq5lMT2WbBzMUsA==","signatures":[{"sig":"MEUCIDZZWYX0kW8/oUCUptStA+gkXW/DlOiPoTVxgDECpIR8AiEAmhMrsT8KPGVe9bgG6Hri710F98esEbXZEil6z49OTAo=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97279,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJga4oaCRA9TVsSAnZWagAAIHcQAIE4cis7ejvS+EwbxBtT\n1pfLnvQDY9O5nKLlu6Gw4xWRX31hWCs+X/5k4piiIjDSLYA7/0bj5H77z/+U\nzlyM3VH5V4WOp6FSefRUfCCUUhki37YL4ZZr1za2Avf0L7xFKZkaTt9GHVsG\nJB8N5T8HnVlujHGl8TyYC2tikA13D+qy/f6jmSOCuvHITv0yBPeTjSSQI723\nwLeJ/1YJaun/j4jxb56WqEOakekACZa4G2wgfeGnv2YOtiUcwCpmjpBcKSIT\nF2KsKqnmHmJX5uqMvsSiQ0DwZZodFxLLyZTCzmn55zxvzVbQNOE5MwZPOQR3\nHNYDZseFT3XP+QzV0KKiIShuHbPDLzgNWcKXCY9tTzn2bdfBgXFiyPJIMczn\nILYWc556ChY+zMo+f1NtsO5jZ1XdNfM6bmqovxy8WfOpSG4Ie52u4zFtR7u9\nVgWaaYUdKHro1wy8y4pXwdkw4g1GFNOQyyibsbYEKXwmPc6DxrXlFlonm6kL\nTCMwuiGwzVMTTPOU78o6RPVg7BNpzJhVRSvQFJwfypLMw8qTkJwlyIuo2stP\nyD02DpmcK7YZrmnSnl4yO1E4wmne8pmOPnSHjJ8t0+BskHVdeZZrFHJsU5A6\ncF5koJQztI+ET2/gskK9TyjbeR9XPZvukLq1FN1eERFH1VHX/1ak12bMFaR/\ndfcF\r\n=evEi\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"d8d12be24958801e54b0a01985ac249d0368327e","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/3.22.1/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-canary.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-canary.0_1617660442101_0.8922130395948622","host":"s3://npm-registry-packages"}},"0.34.0-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.34.0-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-canary.1","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"3e1e6c0bd9f861b1e111c6d6d26d14aae2657e8e","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-canary.1.tgz","fileCount":26,"integrity":"sha512-zXdN0xV6swC0PIiisjQmytwxJHcrJruh1jLKN6OdgaSOypS+75eB14d8X2Uklb/3o5at+zrNkcN18pTj1neJcw==","signatures":[{"sig":"MEQCIBAJksO6bHVwAMrs/sV2DnOwYmB+QVsmOHMla9AyQd1xAiBOoHtKjkns/18U/DHXAU31VU/DoDZ+RGrw5AkOQi7vtA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97279,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgbMi9CRA9TVsSAnZWagAAxWwP/jVekvIqYqSo99YpUgQC\nJEmzTMr6h4KlmzR27PnnF0xCc/1pvUhApIfQSEfJNL5rNUzjTflq66wlafIq\nRlOPNzVNnSaYyVj6q646APu/eXpHm5VRxu5a22p9iUXiX++YrxfBjnvCHNvI\nxMfWjUlbObFyeN7id9/F6XiyCFJ+BydohvJGS+goBkUTMkmSFWG2r6wAMERu\nimiWK7ge6j6eXT9IsBsW9NBQOz4vCcpFTKcd2gFBbyIopXlAnP+2Fas97N1O\n8tfxyqADzXWgakoKMLzGLwdOAi3oKM86JfJ3Bskxehb4gyEQvfjSfH7JCIWR\nSd5nw6pKr656hLUIlkOj6orwY2ddc7m1hAPv0pRQwyPnfqRAERXSDSeQRujL\n1HVpOOdwXyP2OBUx8NSiYozT2XLj7Tz4Li6scWZ7BVoaMcwCk9jyBC2we8/6\nYx+I9fDaJbrRnO1/udEkn7qC/C9GWhru9nIUlklWXdd+hm42fkaeIb4f477z\niUMkMZpaf1tSlyCSPhBmJkw7BEoxddkaUTKlsKhkKhV8wmpUjvBjtq22ihap\nuLVVcKEbg+KQ08LV4BiBHZyJf1/0cBiMb10u0jmM8YUX/Zf1ouoy+17VMcg8\nF3wVX3DvOMbIhgENgAks5qNuIAtEe1d9Zak6t+0aRMnSlmmNhO9UUNjgLKMj\ndq0y\r\n=nntd\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"4d42b462da029523cd112a34c2d8a7a9f11c2c7d","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-canary.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-canary.1_1617742012875_0.5216937900313838","host":"s3://npm-registry-packages"}},"0.34.0-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.34.0-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-canary.2","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"b61388f151cdaa5e6107b73f63d92dda9120c1dd","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-canary.2.tgz","fileCount":26,"integrity":"sha512-8rz8AZJ38wLEkzblsJ87fORhWqE4mjsYnRyWdxp8r86IkZfMK3WuM7/hR3F4NFC71sqOZSuihxPT57ZLIyTM9g==","signatures":[{"sig":"MEYCIQC/I7guDVLXbxPIDXpbyzFE9OX4Oy/3Yz9YtWGKGQ7oAQIhAIq25gC6UBYbM9spZM8bxPW6MG8+cp/dc7y9ap90EqDh","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":97279,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgbgqiCRA9TVsSAnZWagAAxQYQAJqyxVnxzI23j3/l1ltT\nJ2IgzQ9Of3KCdtvz+zVgWV8dwMqDm8Grrh65DDZ9p4srj7zt/+I7EOYiTg9R\nB33kD54kqTrF/DzRXtlzY3+bl4Mvx4Nw5XXrE7xNSDiESy6i9qX57rlun6nE\nYsqHqVPPf2KAc3RiVK0CdSqxWTN/Li1iMCZimETj69uoV6jjTB/XQJokRLPS\nnKLzfBm4O/srcYXCbdqB5uTselw4jDs5nGjjgy7ZpWVeEvEbQ7PwzBd+9R+F\n90tFIs+qq6FOxITP9ox4h5KgohbGGcCVsynt+z77dwGYcus/OfANTBY22vnH\ntfmuM8eJiZlp1E5EG2Ckue0NnOiTqrnYsd5X2wi4qmJka3GvetTBePM6bikr\nuR/pJIL8gxbvnY9ncORXK8/0b2YKKs1lTtObxaUhIOIU7qS6+YVbjKuyr2Ch\nKdGKAZCSyjSfUOd/cB6fKWcPhPCG3kO12HBFXDdyhRIB2PntuwnrGH2TNjr4\ne3Rcs3stgKyBG8Zk43lkQkT/qVNgORprLnC9gqcsuJOSCskbP4orCKhBoXdL\nB2NwpNKPMz3qNhpBNAT+O6rp/YdetGtmD6HNGmsNSu+K1RUBetoRZaTZvZ8w\n1Sh3yhe5RWKNEie36livoX5mtBfAMjmIg85AtQoVQEN3UoHHJWa0yiLatBfd\n3Dk6\r\n=1NQH\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"f3d0f7f79d03c661a14fc57f8c81c8368dd7f7ef","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-canary.2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-canary.2_1617824418033_0.7817248436017126","host":"s3://npm-registry-packages"}},"0.34.0-danger.bda22a774.8":{"name":"@blitzjs/file-pipeline","version":"0.34.0-danger.bda22a774.8","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-danger.bda22a774.8","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"b74e58599b97e575b13e83e59dae13113ca904a0","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-danger.bda22a774.8.tgz","fileCount":26,"integrity":"sha512-0CsMAv5o6OQw+KQUww0633lFvbFzMbrcpWcZOTrHFG1ISclNbvXrVQoSOIcMEUk4Xj0v15wHwm3HxyjUhvT+Yw==","signatures":[{"sig":"MEYCIQCj+CcsxO3KxPnzC1KMZ75FWLTOIdBvq06xWWDM37kwfwIhALTa0Og2FlXZGF0ZUxezFONvz3x2+xONwttf7L9CtTM2","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgchwBCRA9TVsSAnZWagAA9jYP/2GtrZMPAD/yG+jOk0PZ\nklNlicAjlJRstAIz1wI0hyDgBkd7v0ijot5X6YH21xk9C55p/vnMIhtFYo4+\nGdqCkf6fNJDK4y2O/5wXGhbEBQ4or+aWxMTWktF/fRJdMvZ4amTo+CNTwwRM\n9JGSwNNZyCWhkGNDE6B2b2KB4eU3PAL/+qSbWAo7DHur12D5d+BHA0BL4RAA\nZewD75z0+VhuDLFkSSJrF0rzc3ulwNjBHPMGded99Yhf+2NQV2xOTkiZXV2E\n0rFsn/5LWLgQLdPswcwpPxX1qnDuCfo+zCF8mdi5InJI9JZPMYOF56eoFswp\n13BGTeR628XAdxW59BbC20MILEWai/U4Xdi2wzVgN6S6vvFn46OCaW+rUM7J\nY8yuphVSAkbIto8uBfuEft1KPIAKQwsF5xP6p/JUMSBKI67qqnEalMP2RNbm\n1uL81q9pWEyQNyPjKmdwdzNPwNc772K239XUAGuVoUT7oEXxgdAbsOVlmphy\nbZarDrgZXbpOrfAI8qKrKNzcvgStaIPIo7vfslcw/3BdwZu2HMvWMI9Kwypo\nCU+GpEu64RS0lyMhAf6LV3oKEhYZroT4F2trvEv1OmgAy+GC/jWZAbXKRo6P\n20ceppFnACnstzTLLSY0OA4Is/03nVnwwChMmcynQlaQ0pztTS0UUDcdC2M+\nz2TV\r\n=IJn1\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"bda22a77443ec4f1b10a297e000c1c566eb0409a","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-danger.bda22a774.8+bda22a774","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-danger.bda22a774.8_1618091009561_0.2549603074328797","host":"s3://npm-registry-packages"}},"0.34.0-danger.c9dd35ad4.9":{"name":"@blitzjs/file-pipeline","version":"0.34.0-danger.c9dd35ad4.9","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-danger.c9dd35ad4.9","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6762a5bb18b58610e0b7addad5cd2d9a2fe17320","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-danger.c9dd35ad4.9.tgz","fileCount":26,"integrity":"sha512-wo9HGYMiPRnprXakRgyeNXi5ZrCEqDV4LZyjWPNPVeAhPwgD2v6Qo+IOdYdcfL3INYlmofVPr4swRumJJpB8/Q==","signatures":[{"sig":"MEYCIQDrRaxqctbJvtyClcITcIKT4H4e/JTdhG8TWB/12fAz7wIhAOpCcytlBTJQbmG98ucohB33fTtbN0tNjGEphCV0djj2","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgciBkCRA9TVsSAnZWagAAKwoP/jrfpsTnWiwYmB80VpqA\n19dG1cJSaa/q+Dgv4QvOB+eDwOyIbhhhdOtaqz9eXrzOs+aNNsBRQfKGmhUw\nlKRlEqyUyE6chNXi2ocuNHXdTxnUq8EHOBCbGVmlKlPporcmAQqXw+sg6uGd\nSydQVr2d0DdPQnzruMES+FzqvlgIZDpaemZhGdNJtVaGmNubSfmh8brgnk1S\n16JQa5RUTFudsBqG/UnDvFlPp0rtfr5vRtImUtodzdMu2NlPWgIad8KR120A\n8XLje2aCqGxiFT5qWMxVmyVlvLVXScpSIX6n06eHAitz+WnggZDMycySbzSd\nwOZw4NgIf5UyP5yzWPYWLUuOAcF6S/ml4FBGsyMYFBr5BG/V7pVQJobRxHEs\nsOp1Sj8B4wyNxApPSUjBgJtsDHNUW9VQgRYZl7J34MNh8ZDgR09efflGEVwy\ngppvF2xAuSFVX+VMTfUzryU2qE/aSM8Hmzoxz8diOR26kRldgboJ55LzyG3U\nROTAKKoH9UX8fP+jOnacZfseBbgF54VAmR6GcM8XOTaY/P85/DWCKOrgrQXk\nkRuouFvEa3FyrTxnfnCrDqLHComJG9u+nuDEI31Z7JxdWO7lRH5dCG4ofwT1\n4LTU+ut7XXiSudlPI37jrzhAprRc8gzWZlicb8vSlaQ+Nry33HDUvhcGFzcL\nh0D2\r\n=CCeW\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"c9dd35ad4a4042222526df3e37aa4604ef2f47d0","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-danger.c9dd35ad4.9+c9dd35ad4","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-danger.c9dd35ad4.9_1618092131976_0.6422432218931704","host":"s3://npm-registry-packages"}},"0.34.0-danger.625089a0b.9":{"name":"@blitzjs/file-pipeline","version":"0.34.0-danger.625089a0b.9","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-danger.625089a0b.9","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"86eb731dad5d81155b7d987bfddfe347cead9fc9","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-danger.625089a0b.9.tgz","fileCount":26,"integrity":"sha512-np7pgc5G2wiDQERm6YAPVe1ZUhdKoidshp8DQO2Y7LaoM/1YRI8hl3o3JzWUi1Dj3KHQiDqqrWRk33IrglTmng==","signatures":[{"sig":"MEYCIQClcHhHydbJaTWuPCyAXI41/WKkvrs9XE7TcFS/F4vxLQIhAIRwbb7SMoHtwFKcGZLeT6Q4eGEt2wKPl36Ge1iyu7l0","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgciE1CRA9TVsSAnZWagAAAtoP/RYxGlyV57VkexWTeHps\nW5xU/IDrOje4KBkZr09FGREGyJbXG+yE/auY4zq3f4fVlIBu+t9VKX2r/8Ee\nbPsNEjlV6rF6tN/EysK0N4VUtSSUzFQq9WMYgMwQwVYh/5UR3ANqUE1schfh\ns5gs+V7LZX3h+trnueo4g0zbxPGIoGo29VYff5sd4hPsc5NHiLrtb1DeGoC7\nOwPiQgR0nEbGNLNL534KBivM/5EmVb14V2NQf0Pm5/zl9RxWSmADkSRUIzZ4\n/SdN/NGIQYrTt+gVcDZ6Ay1QxiJNkIpqFulEIO5BJ2t2AONC99c+mCv/aIU8\nLqA+megVLUKgSbOX+B8c9EEjg/Kc/nnZidz8SLh6viHSIg3hHo8xP7N9N/oJ\nP1191awM0rd0WtogCgDi4GuOrGNzoHbS81N96c/eobZQtulmBya22Z7mdaq9\nfy6OGA0foO3s0/vxwqdvAgp9zpu9F9uXqX192aaOHhdQCkg7N90QJRRNveYu\nbP1ytWfy4gTReU6ln3J8t/jiVQoje2YRpkRfpUoK4Pmkc9ybx/wx+N5WX588\ntZLSm+cL5QcKDWQVSTW3V3saw5dG7hkr1QcRKr38EHVEvBn1fvsbV3zPCwNf\n1AWAbBoCd51CZmJ5QwRrKNpKrvsS/xL0lm25iWTd/CBYKNTyU2PHz7VXDZea\ncDXO\r\n=/kPw\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"625089a0bac672af7c38e5d08c550688d8f58968","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-danger.625089a0b.9+625089a0b","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-danger.625089a0b.9_1618092341381_0.7262290644328526","host":"s3://npm-registry-packages"}},"0.34.0-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.34.0-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-canary.3","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"9a24b2600c4da3c91d8b33b079e61bce424c29a0","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-canary.3.tgz","fileCount":26,"integrity":"sha512-mByoKq5Ks/aYaPg2cX8j0OZum8Wa5Gf2q5OS0o7+7IiNxo5eRa5tTCPx3IustDRQBQ6WAhE+HtiX6bGB4lbv5w==","signatures":[{"sig":"MEUCIG6TqPyG4B1S/fDfC1PQl+DC8GntPvQGhugvmpOuu/PPAiEA/X9hVGsclf/cET5c3vsXa+yHRsXTX1aYWwkLS+rnDKQ=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100307,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgciWUCRA9TVsSAnZWagAAf7kP/i6cWWLQusnCog8Y1rMQ\n8lJClXQJjcPwQHYXfmjvfXoaHPdxLFeIvl7xK6JyWWj8ilV91a7JARjdIIyc\ncduYpZPhUNRXHcy1EffLNk9nv+F+7dJl/sbSDU/a3hf0A2POgpYDtCcQOOe4\n2Ov7GuSqwP4JO9DnTqmQtYvt0C44QZlIPCHC/BaEQ+jUVcWSr5SuvOWbw9hW\nDG0JyYPe00KHrkqVacTIPjWTrtea1Ezcv9ROS7M1sztQD8cElP/Ja6HCpoxw\nA+kub+50eLXyQeiNWsmrDGK1pb8nJMPbkP7/JlysaEx1sH5BVwI0Kv+5qy8O\nsl8MDcPxvX8yMkF3N9/iPch8DG5kqVmz/qoW26Img22hhdG9kfJNjFIlPLFG\na2IrFrvmPLvoDbkN51JGeUOKiEDE//9C7r0Gs6JW3rdQuQ1l4wlaOpamP8DV\ny4UzWmfW7+RqeaTPFD0JsfbxLmm2UD1L1dH/T/mmq5bargog8rOIBaN9TjYV\nypO9CzoVDemYnsh8aES7BtJgNrALM/EStKivlPsb7iQIEx/rRym21YVBfG97\nZTboR9JDY1ckbKVz9JIaU1yFmdk/KG2Ymsap5vhQiMrUecOKbThC5sSgv2E7\nvUB4aFHG8OQxIgEOhvRa60CDGjW/riYCgnp/Q5eXi6DjyISqHA/Qend+EV4F\nSPLz\r\n=PywR\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"973c2384ec5c40b689c97bcc063bcb924f591c95","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-canary.3","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-canary.3_1618093460432_0.5549007814395344","host":"s3://npm-registry-packages"}},"0.34.0-danger.5dbd9a26f.9":{"name":"@blitzjs/file-pipeline","version":"0.34.0-danger.5dbd9a26f.9","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-danger.5dbd9a26f.9","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"87fb09dd68277a97c6fc221772edf2ab422b64e0","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-danger.5dbd9a26f.9.tgz","fileCount":26,"integrity":"sha512-kRxvyhTWa+8cUtNIWWool+XJx6owNNjL4JMQVWv3/RQPafCvDUdn+NulRCq9rI332YuADNnA6MPp/xb7xKOjLw==","signatures":[{"sig":"MEYCIQCQMQEoJAiFCMeYUafw+8do5HoZTZChsvaUA18KSWYT0gIhAOkall6Yyiy9YlfF5CXK8rEDj10bIDzwfcaUQGBzbli5","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgde3vCRA9TVsSAnZWagAAtt8P/1lqzQbrvPLtyOujBvHx\nNcaUeCW4pL3fAUALRjNGYph21hy1LoxlW1YeBMg0qaPevIE54otVewuU73br\n1+mIFvGkcaEe8M3hLUPSabc2bokjrjCe/iGbxd1xRqSlrdA+1fBhkf2P0Dni\n1j13WUsG5+5aTMKhYQKDpza0njmVH72NiGtTEIMuk8ynyS4OIcGE4+f9cFJr\nCDy5DTqX05puELutWNQBnVw4h69N3ATTXdg9lcEmXWKWuXjsi4tnVdV/sz02\ncL1Rk1ju0lQRBqrRY0kVg+e9clq7zC6Ja/nnjEo59We2wNDWXSIM3w766KhV\nkrbkumhUDLWVT+SiINmoiWhDfOBLMOULoPm+14/NAh1M30ucmaIslJFjZ96B\nPz5Nnje7RhsphNoHCUD1LhfmF5CltEzDMzzvNfFZqeLbJeFy5VTk1juWvDsx\nTzmolRjrCJXYtUZ5kylecZVvKpwZq4tQi9IwIt27ZN7OLRFRYTzDE3C948oB\nxGndRT7BW1YEXmr3+CSfnY59ggV2j1OPIxiDz0J7DnSCXfg1QbITCdmZ3T1Z\nwPvGgI8A4Vtu7twXeAdvcJP65Q2jzsodwlmuR/II+SpfvztoeHCgrpPLjNI0\nl0Ura7NvBgzWnE8kb/iHFHPdBUfaGvaClp6/7R2iACaizGRvnLzAyRaAnSRR\nEU5N\r\n=P/GW\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"5dbd9a26fa135500359c88c90e982735ad97db60","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-danger.5dbd9a26f.9+5dbd9a26f","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-danger.5dbd9a26f.9_1618341359285_0.23646749396985123","host":"s3://npm-registry-packages"}},"0.34.0-danger.e8dd18645.10":{"name":"@blitzjs/file-pipeline","version":"0.34.0-danger.e8dd18645.10","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-danger.e8dd18645.10","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"5476a447bc21e3064f4601cf123f9668cbeb3c30","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-danger.e8dd18645.10.tgz","fileCount":26,"integrity":"sha512-8Sir4iOeejfO7U3bHwLAjAbt9e9PeOkRBA6B8S+enhi9NnbN9HGaJcXfrZQKxrdAG7QHwdSD/QoIDwXwLgQ0kA==","signatures":[{"sig":"MEUCIBfAVLakKLKjsNHuRPShvLlIieWPCKneC+3Z/cLdGDExAiEAnw42KladMM9oVzZpl35BtUtaPmvRkM1RQvYRc/tSctQ=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100349,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgde7kCRA9TVsSAnZWagAAQ1sP/jfgQvjQt8iCgkAUbD+F\nhvnkQzVMPWIPf0t6eFZ4nL6pgVqlGovMdyLAWI4sujofdhT/V3WMwdqqypL7\njQa6Td0R9QrfL2rQZ75ko0DGki8lZ4ZI4IEO4X0VjfcE1/BLodSEdGITbQOH\n9dYad7GJ9SMZ4Q7RwZ491H+AByYmHVUEsODVpTgKjS06qQzAyShxIX6TtYVI\n2jeoLQztcCnzpIvRNQ/uV1jWmmJg6KNiPoVFYxHy5mIjCICKAm2tFtgLxy5i\n6FJ1ybSH+sxWipbLldUQqmA/7TrFc+hKQFktvtQxf8xFPtkSTz5aY1or1FyP\n8hYVbN8Ur6ZeoOlekinowTKf+xe+snXmn53WOjY7c4r4dWHlJNKdQxDX3BGh\nie0lOvOvKBdd0z3C1ywvrBPj9ngHQ23h9YBKDF04e/BgZ8dPauJ7f/BzPpvW\n8pNlm2TSu19DzNFt2ohzvukw4VS7CUXMpcN8SmLE9qlqmHyv2bhPFxnprIDm\nNkeJoVE3pbfid/gV/syVOKYp30pJO8gDqvxtKRHA30py+whsJ+VEimcDjtqq\nTNJXkHkWKKwsLF/oQAWzDJxS6po2rjDmykU9/vosou6qUCaYHElEEBWyXwPJ\nPasUN+3tXg5ozBK0VvM+FgJk1mC4JExG6YUXl0Y5XqiVjwj+C5vmssrNKCTV\nYICE\r\n=eMzO\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"e8dd186458f8bd1c8916c4f50e1a57af42e43a63","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-danger.e8dd18645.10+e8dd18645","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-danger.e8dd18645.10_1618341603972_0.6383228463947428","host":"s3://npm-registry-packages"}},"0.34.0-y.0":{"name":"@blitzjs/file-pipeline","version":"0.34.0-y.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-y.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"650682ba8f2e9c095afd1139caee8fee2eb60330","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-y.0.tgz","fileCount":26,"integrity":"sha512-732/9zaB6QCbvwuzpqi+tL6+902FeACsE4jvVyClxbNUx5im0bxz94juWG+ZBHH8b6U2eKKB34y0pF1jLUplSA==","signatures":[{"sig":"MEUCIEfQSdfhVNxteCtTLiJJnb3c8YwNhVm4Xny3DcbGXpZ4AiEAu9yJ/X+9mN/r+Efsj3MDF4aWZO/PZVPGHXXuP3sEqU8=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100297,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgeG+/CRA9TVsSAnZWagAAkOkP/A11Vv+ZvmcZDPKraD3T\nanwBjaznhFQoXbCj/t6ccmgfD4jlw7Xh+5w8Hb57eAelFlXrALRDe+pUJAG9\nDkBrZnPBndFERLn7CbSiZIqXwcE1RTmkXVGVr0ZKQreLDDrAzNspOHg12/eg\n3fJT3S3Op/rjuth3jJgS1tUiEEq0nO2xe9Bjz8sn+YAerXIZYwjjjOjKUE3G\nidrG3G6imejeWEs2pBfZcSB8Z1s3/gTMoPkv1svNVU+yA/pYG2cE6e0SPrZn\naIXMxTZ+caAzz+4+5W/ZY2JgdUWBODJrSX7ivzDwNIK7RRNjyjXmbgC3Stdh\nsaLq13Tz/uevFhKv6VFPw+fSoUozwOCUXdnJeIHWUWi6X8rAkiZsGikhKlGP\nGsFsU/d9iE2L63z6Evv2fyoOZpCpfaasgGa8qhEVLU3IXfnfzry8+bu3+Drr\nPqpq/vgzW+KELkYpg82nMb5L4dMVnE2RppjGsJOWthyxRt2zNAxVNTRM+G0z\nEL4mAt1xJ0uAQzcYGYQGyAIZVBW6JHyveOwhJwvvkROCcO1rCaVccLEMeoPU\nWKNjW0PSJDyqPZEtCz1k+bmGTzMcXeSoqt5ppEAYZW9gVyz+vU17277K2ZTr\nbJTPEhmpug1JyBwDRVxbLKyPJGo5+Dmp3WLacP17Tr+7OunxfegryyfNj9g1\nPYH0\r\n=hFCM\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"b6d264c294e3dd91f380afacc31c9a3625a2ec08","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-y.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-y.0_1618505663110_0.309248137552818","host":"s3://npm-registry-packages"}},"0.34.0-canary.4":{"name":"@blitzjs/file-pipeline","version":"0.34.0-canary.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-canary.4","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c2835688fc4d3433536cdc599810290b9beba520","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-canary.4.tgz","fileCount":26,"integrity":"sha512-d3utNTiirDQNmn7L0NrwP6yjr054d2YNQrOVETThM8dJSszH5385n6zKYyQn60rw785QYhJIvhpIQRTA0AJbWw==","signatures":[{"sig":"MEUCIGfG11D1PVho9wL7sVL+Rtr10CezrNb36LxVwF6T0xWqAiEAyZvUUbhz42IpxLxDt9kxhiRBmZGTQ1vhvd7VY5pi8T4=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100307,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgeHGyCRA9TVsSAnZWagAA8fAP/RynV8dxmLLh7Hgc3JH1\nqrUo3y+QbbgcXoSSaSUQARsXH5PNsE5bCUxVckk1yoO9dYjAdUCNgrZYPt27\nwfpGS94wW/1x3kLamUh7sfbl7iAcSIKPZvVZrTslVc2w+pi31H6YDIKmT0pv\nwnehCXvXnmSxRMZVjvdVsU2G+U/Z9KeKZi5mpi3ha0xuRyiATT/DqCTE2nx9\nb87QktTXqA/KId2mYiuGcMkYPn1bAVlK5I+hRoXYaWZyygC81u3E6gmtlwBu\nmNIA2hErzExvZhb7YnaBPVXn1spHNGHeP+4dxbO4yEJfX7xkLfCQXocsSCCN\nz4g9LmE1M9iqX2I+WK1G+Cq/rw1AJd6ZX71mIgjXgVCayPMJgcwTIDfR8iUw\ngiIOQWYVxL5RKaKjB+5L1l78qe/xrUuX/SRCtIJjZj311q11uiYRJGfvGwYR\nim4hSghH0NcHen3al3a0p14Q1L7ZykLJ/P2Ta6OevGCn5rEjAyUxxwrVIzYL\nakMaoCvvCrugR7P/0m6AQSFnpSGe/l2SSVJXAxCNdMPowp0lmqlSFN/BSiu5\nPwBgo+BSficwMA7hdnw0xkLrexFWKqRdwew00/yh6r8YxgPUzAXQ9mZKj9Mj\n++/gsrRoV6UjiP7MRAgOVJR0hE+r6+hRFaVM7Stfmgiwan2SSXYVZ+CDy6Ki\n3hlY\r\n=tL7m\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"fdab34bee40fd35bdabbadc43a8093aa1776bdaa","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-canary.4","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-canary.4_1618506161949_0.585922021799052","host":"s3://npm-registry-packages"}},"0.34.0-danger.2a307c801.1":{"name":"@blitzjs/file-pipeline","version":"0.34.0-danger.2a307c801.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-danger.2a307c801.1","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"e4b76f83d81105dc39f8b43d7098b5826ed58403","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-danger.2a307c801.1.tgz","fileCount":26,"integrity":"sha512-jMWpWhDfxSYAxd+RfcuBjcRHaOTmBCL5yPwaf8MHMcGHA2T2q6e0PMCfGH/Xxt0mryASP+bypu/lkD4dFbPXLg==","signatures":[{"sig":"MEUCIEwtFc8RUfV3YLmHQgc2BXlwceoFw+ka8iMHpcSNNXJnAiEA6AG3RdiEF/RgfjnT5ZkFMrBfQSWLTBUGFS1Z8YPzrLk=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgeKKmCRA9TVsSAnZWagAAdmUP+gPqDSHL7U9I9jMiYMbb\nowN9AhXujw8Aau67/7LaqSwtEL/qhOHSJ3dWtBQzhKWa2g9ldz7x6qXplGdC\noNVscVXHHwN4e0AlKWJl28WFB1StVaOd5rmBgUgYmyRBptjAi03qG6HRjt8q\n/TCZQ5AEbMw2xMASkrsCGGEx3GWUfr4RJzzGuMpYx6EtWRYk/ggtYRXIgtjv\nGJbbNvMHPvyJBRySfvM76zLb5GdEc49u2Go54gTx6yaMGbS/+zuDYEMawFNr\nR+ysXQbU04TO45CYSsMyXQzh0woDKvINxfzZo3ta0Jc/SW+FraEjE95fhgxH\nduqdZMBonLqhRVHqt6F3x2DiPZxT7aCQDNkeSCAiuEyRTJMXwKbhTiZaeZ51\nVIuz6TsrXKvC4hVpua/HKxlHIrKO8K191W1Rmjzi5DnHjfOTi5zblPHtzQzW\nyaloC68LjSHbXZtYh6Wl/j7p9bOCKyskmyFNRu9ZjwBs+EEyMNfxqLr6dbJ/\nJBUlLQqVSDFluQne5bZJs0GMnwoEvFmvBbb56jrfNQ7UnCsj+Yjh3YjGJpd6\nPg42VAdwQkh/HCHpZC60sdTHG3K4f38KexFnI91uhGtJ8IFineIt0Pie0wap\nuAzanfg6vj2PmjzpNmXEfAz7aO4XcaStDDLHJS4ML5Je17YB10ed0oYz4Rgp\nibpr\r\n=BrFz\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"2a307c801a5573c0fb1cb8cf9af0c55e3d573f4a","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-danger.2a307c801.1+2a307c801","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-danger.2a307c801.1_1618518693978_0.8590781879918687","host":"s3://npm-registry-packages"}},"0.34.0-canary.5":{"name":"@blitzjs/file-pipeline","version":"0.34.0-canary.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0-canary.5","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"13975b946f3485f003b33fa489007e0e3dd8836a","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0-canary.5.tgz","fileCount":26,"integrity":"sha512-kRTmOo0ArgVKXoHAS6/b0E7JHe+Sv4vgSb2b+HGDwqHnjMDgr1UOjfMjKTywj5NMCWfXUYz7eaNnVD+HA4oAtg==","signatures":[{"sig":"MEUCIAX96ooizONn/Tyr8bN9qRZHOtOM9xFC42vWIwUDgxbtAiEAu4zlBDnZiT8+43EDpHSooNhr6v5pKqe54vQ6FD2Z+Bw=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100307,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgeKZuCRA9TVsSAnZWagAAYicP/iWGjFbipdIeblB5UWvp\nktyn6Jr6JrBSSPdPphXpqFue2rasDCL8mo+88X60sqv9861nZPnHWiD5tG0A\nG+wZzgcytWBbKbuyQtQv2zQzlBK7eEsmzJ+SOq9FabKFGOTZMWwVRjRmYZEM\nqogQv5HoJHmMq/zBkAKHIkn7gwfQCRVtNleW69Iq2tXvAqyq/SZG/ArA+xQ3\nzZ0iNjvGlYz7ZtN236mSXYnWv6RbdJiKpKtVuqVX338JisaXvu58RdsRwrSl\n3ol9QNv7Lm1vcqGjdn/Isj6jWDNZRYtcSnv6kS9RVgUtkEIQt4Urln2jrr52\nJVFd4J01W142YdHG4VL2GmX/HmnmZzHCSyGKXAYyxd3/FSCZ7rK4wA8m2rbR\nk4zALPQ/atkrEAnQaIHLKCkQoWW/vf6SAzDYaoQY5edQEFIt1O1nudg0EF8y\npgf1Y6sEUe52RWNHMnC4JpValHbpl57QhP5xnt/JYn/SKIRTRIjvNLeYscvr\nK+4g6dKUoRfQc4UBEKnENRYBIVr4fukDyM1WTdONxqOilcDda21hyYnYK03G\nOEB2kqrnRzBknBEwOqOoB7rxRSe6+SR54qM7mohmZ41QuUkalSVWciqTZvFg\nU0PTGE+w5Ll9jOesQBWnXTQJLOaPom7oAlLmpqwcmVn6XZz9Dv083z85XJrN\n6U4N\r\n=4a99\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"617ab8c564b854b760995232bed7ccc1389ccb68","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0-canary.5","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0-canary.5_1618519662100_0.7013501793250625","host":"s3://npm-registry-packages"}},"0.34.0":{"name":"@blitzjs/file-pipeline","version":"0.34.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c6c6543e84aae1213529c365c0ac3fcc88e71343","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.0.tgz","fileCount":26,"integrity":"sha512-nyAT9zTy4xZQlXu4MLC72lbedUVuKQ0pbY/bJeFwcZ8kKHD4tX3YZNGvW5qgBVQPrnk2uUB9uaGnkGpTLfBq7g==","signatures":[{"sig":"MEYCIQD0lZBNvlEbarcTCfKhtRAK1mXAbb2gPFqse+XmycJSmAIhAOmQArbPDfyOulkUilh/5BTx1CoLjOHXhShCC4Pg29JQ","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgeejZCRA9TVsSAnZWagAAIGAP/0xAJqJh5eNbm01B/cjC\nv5hSEwa9NivaXAVc4WyJ32E/MAS/4H7gvArAY18NF/3TRBIAjKgT8MAVwZ5Q\nQRQNd9z0DVk8HFMkv5tbrSu8k8KZkotvbb7h62MMzVhITg6yCYpT7nQr1sk5\nfh8IQF2QaN/yuPQVG6cowTRUiFRa1Lgj3P5hBM+FThdWwW6In8G7GUE97CeK\nFZ+PtXpf+nu+HthrE/CsbKG1vy99frglL4dHdV2U72r6VvHZzPq4Ybifa5pM\nc/6VDUlNBJRS8vBRMajGjLyZiAjXSFxCBCKRUuNjbnyttCgM1Uvuq3XOdw+L\nMY/y6enBQ+ZanWZgV3AmgAOFENPy5OR35+Yz+zoF/JJvaL8W4gjTLUePQwrP\nA+XjC9G55RzAppdD1VL/smtJRmy0XUPZNYGetbquEEW0KVsVuTcFdzci/VSI\ntJbwLbSw8u3EDplPpX1qOwhtQh6WrU0pvhVNgayA847ryDHcVtltSWz4+fJN\nEoBgcx0cc/dq+m1DpD0UDueWsu7HjnXq1N6cEc6PAza8UQLSf8J/tDcojU4F\nPuftDpoxOuOddCVNIbZYCMzoMLX7txukf8SzgIWrbFZp69cAXdi9dVnWAo8d\n761jiDsiGlSVa7dewDeDw9W/CFdw0VdygXXApd61L3+zsY5Fe1US7nIJPTEd\ncaYH\r\n=jU2S\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"c335a1d96874c7acf41ad6d16690d9701eb12a0c","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.0_1618602201025_0.16292099974239393","host":"s3://npm-registry-packages"}},"0.34.1":{"name":"@blitzjs/file-pipeline","version":"0.34.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.1","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"11c9369e4b20f86cdb731a50d9fdfff004bdb4f0","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.1.tgz","fileCount":26,"integrity":"sha512-M19bc8oOt8nszKjheqrEV20sVRE/cEvhuLM8mjz3uEslX00QVwaEDKTX6ubZJ4mfsuHS/ACGHjExC5ka79HZcQ==","signatures":[{"sig":"MEYCIQCl6gsz+Ac5KFoKqmAdJunJyl9CguzwJVzLlJhtRYbvowIhANSxAV7ljqq3CiCN4zTfWMQ+newLMjnpToFsSTrFqQPY","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgfhfqCRA9TVsSAnZWagAANrMP/1ePSaqPiK1j0FD6FcAE\nkssCEeRTI1vo+bX8tinoQaKAbABXcus2RKv6DAXJWmSHQLGSfeVhaWQrDA0A\n1B8kktqwwyKlciMktsEVHNVT9CFAYeVHeFJYEN5NNdk9+wiScEuVCyUqK4pB\nPBf+9oLd/ey9F7yAQpbhgQwSGLIqhWRHcJ4f/ldi1lo0172Qeq3244GNvyGb\nuzoJ+OFpMSLbbDyOB9B63ee+w+CEMBhYOn/6KaBhInKWSCj+AYjeneKFzSdd\n5seJZ2okwxlEX6rPtn0gsfRDc3QGTqRv3AHlPyAe8eaOgAHqoirE1tf8Y2nN\nf7z0V4Xi1Ez0zA72+vFbpAXxZYpjPHGxmnCMdeJAsk8hTGMDRoTGtv/iZcrc\nS0kJ1Av0trrRAfGoEzPt7GQsRNHcRR/C6U2hpcr+jsx3vcxpxNMlQUmFoqiG\nrNVvm1eqC8BgPlxhttG3pe4v1O8OobUOFCov1vtVG822K8NRCt5yQhspVLNk\nM3h1wNz6v0oz/NN4c86hwpVPWORP0vVpIFab7hRm8HpKSjc4cCFsbaId3zfF\nV+EYTaUyeYgjzjT1U7FvDynvxFDS4qYDkYnLW87xpOPhGJgx93m91DRgcYz+\nAys3Dbqc8JvIZ90TKhI/oyK0uD0bxXcs7qqALaCPdx0D0dixGxpmzYQJS7qD\n6YuO\r\n=Us7Q\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"6a3c2fffebb9c0554e9996500ed9f87e96d7cdef","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.1_1618876394563_0.37378771066409233","host":"s3://npm-registry-packages"}},"0.34.2-danger.1079ed84d.3":{"name":"@blitzjs/file-pipeline","version":"0.34.2-danger.1079ed84d.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.2-danger.1079ed84d.3","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"8e844e804fed507971f8a5d7ce0d5ee063fed91f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.2-danger.1079ed84d.3.tgz","fileCount":26,"integrity":"sha512-k7y74HqJ8ASsUvQKCX6izpB+hpIj/i+a9vJoPak6HTDbHWTRn2+psJoJQ6joLKPR6ERyqRkmaoQiIgwZTkHUYw==","signatures":[{"sig":"MEQCIFQpqLEI6RccaivSe5PAzujB2wd3HertGdmJ7RrEbb0oAiAWjFj/BAeYWkSLcomCXJ4XSWxfe25GrkpCHiXhQc523A==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgfvhwCRA9TVsSAnZWagAAansQAIQDFfZ2pNDwuUKRjsTK\nJax9COZt7xVaCiBGix7Zh2K/0T+9Qt+Q9dB23OHIESvBTucj8UE4wtIdIxB2\nWSnyd6WBb4/ac8nD93BuagQsJ7r2e9X3VzWHYL7hFZbWuolNFcm0+aYFvz+U\no6IPuOr1TNbwnrJhjd+GXLZbgZrLw+xxn7/srIuBq+rxbkLwg9YEpbDn0PCr\nL4vP/9/j9uou/n4jj8kTfaxvuZ5sSsuf6yrM4PQ4kvZoCWmF2xf4EMsyt+T7\nXpbFVZFzcCYTxhZjJBXdHLhorUt8hsZO/4NbLiCaVgWkUtG1c/usZCNUAG7c\nhkTYM0HJ1hHgZzj/TB6m13VyQlYpoTE4upiDNpA2VPgJF6+u6zGcoZgyArR9\ndM8enLyFxz2Tey5SY2yqEWljxggeWHZ/DjIV5484qAnd1al2A+XYJI/LCsHT\n6Fkl4WcfutlgwrLYtzTc6shIXHMi7JKzwhJJz7HSYYs5obWaLq9Alowiudhm\n/3S22NlQKu6JAs3M2DZfH1LYAMeqA7mnQ1Qccbvpy4i9TSkHABUMWDk441Ko\nBXqPf0HCLcN8gza28NKz/x9xG8x7F8dYj9AODdD8p7jgXbFi09KjRKPtZyOS\nvx0KTGdM7s32rC3xqO/bAEVdCGReYfC0Px1XHUfgpW53aD/mYlVJ4UsW3zBm\nH/qd\r\n=hMXO\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"1079ed84d03fb37a514c020ab6ecca783eb5e97c","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.2-danger.1079ed84d.3+1079ed84d","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.2-danger.1079ed84d.3_1618933872491_0.12005635546489701","host":"s3://npm-registry-packages"}},"0.34.2-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.34.2-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.2-canary.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"ac9cc36e4ba7d14415ffb67fabaaf55638434b80","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.2-canary.0.tgz","fileCount":26,"integrity":"sha512-oSm74EgdSvsm4366sWqGOXSJ4VFR8Ma9OicxTQVfgXx2XeqN8HNGTdY13p6qY7TdqNaTA+xnJypF83gYi3PLTQ==","signatures":[{"sig":"MEUCIQC3c1yb+iRBXJYKJfhAH27u6aYV5dSPS+E3VBLQivug6gIgWqW8YT4h6y2afw0qMm/pUc2cFGElsOBHTF9A9tjou1A=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100307,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJggEu7CRA9TVsSAnZWagAA3NgP/0bHQVPbLOzPB/PT6MTR\nmNrowpZdA2HF6ZhB1A4SFYMbaVLVNVijMJUFUIbpD55eUhYvO+WZeYpBuceA\n4NQngJ9uT2uDVJLrK9ABdmgL+zaEw5T1EaZb85Mi1livbhsS2xwPi8x+aJK7\nFQGClD/tj2J9yRM8hBeGhaKizUGZBqCZVOAS64pNE2e0Q++Xxltp7pDW7Y/9\nulCNktuiJ90iQfVd8jQj3vji6UidhGU8jR8G4Rwz/3azkyYDcam3avbNVB7W\nzMEQ8RVcVNlPcT1GkvKgiJE9oTIMMCpZsDsQQL/0VzawB5OlwVngwHBX6hE/\n/ZUzMFhhe094vp8gT5pubn+hh57LfwLkYXp/+tUnMoJ4bH6OWHvpFkRnikFH\n95pctax9OTZCiASO4MIazbxL+/ALu6dg+cE3wME9sBtcRU+rCR59HfMtUfbn\nmctAZeQilDcAkIc2o1QOvdj1EdoJoNDuOMlZ2Ks/bZD1tofo0NU8WqoD/wvl\n2JVRKLKNLuVYPBQn1VXjolRIVqOu/phuDNlTzOxb0lxh4Ibv4JaTICxHeBu/\n3yoQSSc4hrvJUzbkr3fGyPDZ2McVwlpZtabo449Tf2Ge67ARrfpY3Gw394tE\ngjuDOQx78ch673ROYPBBcGaVnk7h47VzdjPa2Yt4uHVh3NtM2czx+wBQVlUa\ngPXF\r\n=zQci\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"6ee30561e7e89447bbff6445dc3c881fc70572f3","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.2-canary.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.2-canary.0_1619020731087_0.8880340184435609","host":"s3://npm-registry-packages"}},"0.34.2":{"name":"@blitzjs/file-pipeline","version":"0.34.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.2","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"bc65905d7a0eed6d9c684a8afb57c125bb8574dd","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.2.tgz","fileCount":26,"integrity":"sha512-UJwgU6RWTczMyuriGBpPyRWTMCMZ7c3nPykPamed/k6oRXh/hMnAawaeT8EHIFfs3hIIS+q40/lkzC2Taqhi2A==","signatures":[{"sig":"MEYCIQDlpyYWf/5J+omr4aoy/8ccdAzg+kBIO2fqW/UK0SqY8gIhANUoaSdcw4mK+4rAqPjEl/MM1W6wXxs21YNGOceMlIT5","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJggEyoCRA9TVsSAnZWagAAHEkP/iMDWbetlcn24OnS0NxY\n7Pzyq9h+/Y8qdPSDmgY/bqy3fIjZh3N55kMTPJYdGelNRDW1pBDFlhED75Gw\nFLO8qsfaIzvsJnuorhFHndw1AQswIm6KYvhJyHESNlrQ6HNn9L3MSSjXmHkC\njckB9ikzjOnKTmeOHUdNyC7G5W9+lGQZGpACZLJIuoKX/VWmw/IoM4Ip3wts\nF0m2CvGg3PAxN6jNZ/OexAERgJJbFHZ9cY02rk04rPLIwzpaqzLUrx+zlXO2\nfkmA1njf/al/jz62kPhpQCyvOXj1SbfFoSl4aX4W+UurOa7sLMUf1Jk2p5LV\nrgygwRBHn/0bX+XfObgqYVMj2Kg8dXSk52qVH8w8U1KL5BAbu0VPg9v4fbot\ni3v9+QIWk3JqNMZE5p39DbNDC3fiQhJSOwDU5twtselcZeQVymtuBeWXbhH1\n1Qh8FzVMBkkYMRC/6Q8p43s1q0lIUSnMCONM+uLk8lYm/Hv3Go8Znmvrmkrn\n90EqmUH1+GfqxQXJngzqbVkv0IMKi6WcY3+abZfHrMS4QT8hzRDQuN9lFa34\nPJ2K8lfNvTRlLGfG6E9GvxMz4C3rIqozRVTeKGCoY2ORLCgMjmAMMQcpOfv3\n4TUmJmNziR+Ku6Hn/KrC1hcmNmaUHPqj+84LcKHM4Ip4mMHl1iDaLoX4CH0Z\npUkT\r\n=OID/\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"9cb37aac9357e0bd2f1f1cb4673b8c9407ae2d47","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.2_1619020967955_0.43287023469855845","host":"s3://npm-registry-packages"}},"0.34.3":{"name":"@blitzjs/file-pipeline","version":"0.34.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.3","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"10c40e8906683fc5e936b27f04b981f7ad4c5cf7","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.3.tgz","fileCount":26,"integrity":"sha512-AeSAypDniFuXC9xUbz07daFAbQWBIUzlz3M6tCWA2lWFBegPDciC/3xKeP/LCHjKqSVpA5tQcR2qVsoxM0R1Qg==","signatures":[{"sig":"MEUCIBtmDTqfyaiUn5twj2Q74JM6GTEfKL7og6jNKAci4SKqAiEAsyNlQ754txOgQkRTJuvFTGrK572jvAKatIAigLHedo4=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJggFAACRA9TVsSAnZWagAABYIQAJucPSr3a16WaAZ0X9PG\naz5A0Ap9kOY/LHhIVQglrEwHVsYshyNP+tQHl2YC0UcjKS3Vnor8fITpiSmT\nvTwDeZOZqaot8Tk6JvR4dAl7xfyZK5PcFyfrdf0svM884mRCveC3VbojSI/b\n6k0iNOjKnHCWtJedwpDR+B9h3tqrKZTUKggE6mrtzilb6KYd35L1Jtfdi1bw\ntUzAaCwzXXHdVr9ggo5tiLVpKD996xfwWcv99tiOLiA35Ibqetih5kRUwZ6n\nCMZgvfNGgkdV4vq99bqKYmj94xfSusIfqN0TSCHje0on0V6aioCt2bv0nXLk\npaIKv/a29BeglB/mYJyCqreU0rKvrKUeai5TS2pghz+qlri6z40brSARe1nl\nsivcZqX/E2FnX5MPk+8vKLfCskJQI5H4N2F1kU3uzy5h4KC+dbno0g4HQFwx\ng79v2o+CgCRLqEPIZERkMpLCU1nJW6XNtbgjEbA9fSWBVaclcTkBwOidN0Ni\n7kie+vBwH9NzNWfbHCyKbyciceSYlHXguTYxffCYiZNRhTDl0p79Kz6OdHsz\nQtfbVt8KwJ4sP5jrVv0qv39UzNqN6SgPgmclHUPj//X8Pqkbi6pn7mH1SV89\nEhNqZ4WYxH9woemrdRb3DH/vV9oqZKI0yr0u2BbKhP0nKq1KLPd6YcOGRS0J\nlPtp\r\n=E9ga\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"ec3f871189878f63965c6d3e0088f2e428c1143a","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.3","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.3_1619021824001_0.2791760387898581","host":"s3://npm-registry-packages"}},"0.35.0-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.35.0-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.0-canary.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"5ea4131c862b9f9c77888179812180cba23841e6","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.0-canary.0.tgz","fileCount":26,"integrity":"sha512-rOvqAlcJVm/S/ak5SDP+PV0yOvlZINjRLv0GmuET8ASQCYKQnvvDt9zt5w7o6mdS2YngzcDscQxUpUkhJfmjXQ==","signatures":[{"sig":"MEYCIQCkzVDvs6ynQs1LVggAnFLhG1pDT5YRlx/PfP3FzAH7uQIhAJFm+Zj6pUef800ct1nXikStv01zr+x33jAjLkHGmNxd","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100307,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJggJ0kCRA9TVsSAnZWagAAF/gP/RRXXHBD6UDwwL589Z/j\nOxmkTbOp5m4QN6gMvxIqkRg1tc8FMYt9qQ+H4bXYZApkD+/2lE3Hs2SOvsRs\nH9MrlIADeFd/7AIgSBRuYphLK/3CD551x3p+BpVEgYK91fajYdGKL5O/Xo7/\nlGlcZzM5HsbEsbvsCl/4EASwHa3haEDfQeWTWxCb8dC4IjtCnRWe4eVDOLo6\n+DD84CBxJL7ZW5IR+UuO7NIIt5qGI7yJ4JppqxX6uc7OhuQAjTLd0vDpsQZJ\nl3cS7fml95FIZM6+1E/5ZQdN9beIPQYT5STGBNeffUgD4hXcMDqr2/fPMjhV\nX/GPVET619VEPQgmRJKncn05mNdgMvCe9DPKxATDJgeq+u/bkePjM+K79DKh\nEcf6kMLB3varP6XdFkcP8j+olGGpyTbDwstVz9eFrivgGmfT9MQlHXKBw/sk\nmZBd3GrigQS35b9fj0HVXxjX2/0Wn2uhOnpGCHu2nYhSGz/71Q58EDKAOCyg\npEundeRfT6z2vV0cnm9DgXaiAva0tqzWh4cd3tkTlQYWZ2HeaQ2EauLtmyYs\nSeBDwL3fEAYxlgxM6mHUUk2bXsj7UixujZtwsxs1hGvsEbOHZveemwcMg9oN\nlTdbxRqy583hYkkzFGFnvjspa2W0Cc70pTGOZqiZNeDCYockmnylKa+C4yLO\n71nm\r\n=lOiC\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"08afd85e7b8b20d0d1d4c42183095a6834b1b8d0","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.0-canary.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.0-canary.0_1619041571979_0.10177715252400787","host":"s3://npm-registry-packages"}},"0.35.0-danger.fa264b8d8.4":{"name":"@blitzjs/file-pipeline","version":"0.35.0-danger.fa264b8d8.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.0-danger.fa264b8d8.4","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"9df1e0a4c9187a2fcac471fb46a6a51a28226d6d","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.0-danger.fa264b8d8.4.tgz","fileCount":26,"integrity":"sha512-ZlWX1IGZWyk3bQ76HeqY/o7yYmxrVvCkTQHmuuHpBJJdrNRNDOwDzzBAFlQojgleaoj8pJW9YHCK8SnScI3Zkw==","signatures":[{"sig":"MEUCIQC47fAVog+m4BS0Ky0loQKjbqcavky8BCNog1iz+QG6VgIgNNVPhnhTey2kXnNwawEso11zVa1YehIO8B/T9kMKt00=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJghXgpCRA9TVsSAnZWagAABf4P/ib0fKSYHmxLkNJsbiKA\n9nJS0L8EFbzk86Xpu0lVbmIuHBxj1xW0znsOv4hbUl6JfDhlpE6cWwNbXGi5\nBc+0JS7Jq3SPWSUJHC3+TFWjbTQSi7k4cHmBx0sIAESKc5EFonvHx7JTKITO\nlKTSJqW6QuTZSWeaoDOl4msgSjffDnDFeuuaJnI7GlHnPiBGv9IWcuHHrYjC\nT/W3Kh3xuCHMSEw3EkbXNIznbCCTaYk6xCrSGJ4uV7y2NkoM8KI0evVCoMs3\nN5vjNt0fFy94SW14IrPMTS2vrnPmteQNSbxUNnQxDRRKnRsPcgQgOw5yQPeK\nkDaE6We7XtSfTDELMchGgLKsXXRtLNLWIWZD0xqkKkLQ1RG5wU3eW/McTbQY\nfn1aOvsHRjQGyDm7332WAKcUGAp4pdpXL3qrhpgNEXYfEIsvPj6P9bithOWZ\nn+6dTRTaKWLrdjYBlYBjd6GreU4sOd0xVk3sGwDYqLYVu0ETGcMrhNrQjH1k\n6fl1f6JGe7lsu3Ms1KWsc7izVHw/Iv+nfqsG5w15r5Mr/FJDw+SEOo8u3WNj\nDsdP2pZkft17Kk1J19QB/M6VWfCAbSbtTlrPv4fMGRQDr/dnzJ6xwJ/soyrM\nrCHXFd/T+MS5CwreyZJZP5G+ROXfURqgYGJfMGremdyFD+sMemQuxOrcQTsO\nJ303\r\n=HZXA\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"fa264b8d8cb2a39faa68188c3ba8cfe1866a7300","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.0-danger.fa264b8d8.4+fa264b8d8","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.0-danger.fa264b8d8.4_1619359784895_0.29876123209144967","host":"s3://npm-registry-packages"}},"0.35.0-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.35.0-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.0-canary.1","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"42175564ce35550df8e16e2d061c451757ff74b1","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.0-canary.1.tgz","fileCount":26,"integrity":"sha512-lIs84xUExr8+RXmfTlPhXiSmSccMeUGSHvG7ZgwKzHjN4GJS+lhGDhTNkLa+SGDyRnwnhpFqGw2QFrGC4vB9TA==","signatures":[{"sig":"MEQCIAyDHho3PpoDMiZWvp1/sRMK5zDCRpi8Ulw52O/MwYouAiApQM10zKM4T/lok6C5otEaWziMl7LZHoLlJpnkTJYJ3w==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100307,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJghxCdCRA9TVsSAnZWagAAjhQP/iDKl7/z6Ki1oJvnl6ri\nhOM1sdhOLcTL0vRI+LmroA8GO8NmO6DeHnyMaWqklwarQFcAQ2N9Zpq63+Cz\nLYOSKbUJ00eAo57X9P13f66xmcZUorxOBD5e1tD3qPXyetJgLgONRZM0i9Bj\npu7IeCZ7xdwG4Twb2PGYtkfjLscOSahcg2As2memLvY+Y7cq4qHfhjLCkILq\nEsflGkKqTLGXAP10CnjBX5hgX0xtsGd8Fhir+2an9XX5d4NN4HspoGMon84e\nn5+Zfa0xl5E4RZEX2xbQhAcbbH3jpECkW3R+o2L9zXlYmq7DBTDMh9lS2HWY\n+/HXNmbUpljoRaWjNBGb868QvItiRmLPdImpFrrnp+D1pVIiBuFz0kOIx2xN\njCZSB2SF9Iu6JdnrlaR+jXx3F+S4nX86mVgxD7WpNfzfqaj1MZIjnZS0taT0\nVbOjX8g6VCrWKFPrEXzYHLeRNEO7P0n/n0AAJoUwDoiX2yc5RT86PWAhhhIz\nzQAUOv2Yr7E8KP69jD7/R5OBOUDLM1pm9uY4HFIXC21wvEkpauVj3nfKVmbl\n5ws2c/i4FO16hvueA3Ts9vj+LlSnPwGAp7hUiSdr+15VlxpqeApMDugRreC3\nWiIEazY+RW9nXd4Vb5spJi/wf9RJ1bFSRxv347S50XzpbC8auLH7pJBqqb5A\ngT4n\r\n=KTrV\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"03d3bef90725ae5dd622e4fe864b88bc9c732d2d","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.0-canary.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.0-canary.1_1619464348706_0.8052026948489925","host":"s3://npm-registry-packages"}},"0.35.0-canary.2":{"name":"@blitzjs/file-pipeline","version":"0.35.0-canary.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.0-canary.2","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"185d62b5ddd50190208ffd21334ebaa3530eb483","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.0-canary.2.tgz","fileCount":26,"integrity":"sha512-ecfvwrqX3RjanRlmO/ILUpAWgChAV9wlp9SN2Qa7e1xmqrhs0iqojxh2Pv1pDUH7He5VAhS87kKPc2GiuelGRg==","signatures":[{"sig":"MEQCIGkpRjPvQoT2VsFunttzT/Eo+0HV+nMZlfEfSIXMjXWeAiBaw0NTTZX6La4sO89tI7X6rN9CBp2VkrGp5ao3Y6lrqA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100307,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgiGr6CRA9TVsSAnZWagAAY78P/jSA5/g0iTl2hQelWtl3\n8f1X7IyzX2dU+TErrOoYfTVNzaBbVJqJtLwf1vAxZ92aD/jhdRIMIFBUNPsR\nWaOkJLBe98yzipR3akKgtH4zzraLmFVJw09qep5EU9OMN0sitp2m11dykh8f\nvRr4I8ZPgQOwKwuBHxjNzhofVBJmObSO9HciHTC5neirtOdKzomKs2ig7FOf\nA30DMHIykhlui1K8xH0QSEWoXbSyhgHTqtfGyvaCm8gWVxR0wyEe2YZiLzJb\nGEzl+pSqYPLN/WUvkTAF2ODey2gF7K8fUBfUkAr28LSfMVO7UW/T3cbwfUeG\nVi25jIBfWpD70lYuOFuWsxCy4u3FOcgB8GvjUZ9bknJpwO28GaMNeimKfsa3\nhgFrzHiV793lbD5L+4C2+k/44AuiH+4Sasm8APJviFX8YAyYTJoUnxwvqp+0\ncmbXuyhyRScVwYzViX6lS1CIBvBHOmRuAWe32G72cqzTL0ETpr2xYatNv3ii\nMO2JMftdqlMmL3YztSLogPM3PAvIX4NybnN3/r8fQkrVtUypB4w/II5Qc4CN\nYjR/nk86BMw+GRhvKV4bMaAG6wf/ky4z/Lpb1mNME0qZhtPWs6V9dtLjButi\nMWWQK65cE3pd1/uutBRF2vg9YFEGE7sx6HylIPtD+REyyLLzRyF4um/Wtw6I\n+d9S\r\n=La4o\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"b6b27166fb256b169650b3d0641c7ba94fcaad7a","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.0-canary.2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.0-canary.2_1619553018306_0.0491110572655975","host":"s3://npm-registry-packages"}},"0.34.4-danger.f06f4b853.6":{"name":"@blitzjs/file-pipeline","version":"0.34.4-danger.f06f4b853.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.34.4-danger.f06f4b853.6","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"d741c095c690be54d7a01e1254a30065f64747e7","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.34.4-danger.f06f4b853.6.tgz","fileCount":26,"integrity":"sha512-O2HelI0LRCP08wqFlBUIcWwJvOlgSJZ9zyaJU+GeVr2lkOicd0aT8Dj1DSrFThjYj79AGzSO7UEp4/6Aj43bVw==","signatures":[{"sig":"MEUCIQCVhKZjqI+dRvUZASalKi5x+13Sl8ZAycjzum8fnRcSeQIgbsrhw+yn6VesCQ2NxDYPOpTSIemaOxiiBs8ZAQeoQ+o=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgiuo0CRA9TVsSAnZWagAAvIgP/3zONBbgJSFUi7Uvlpj9\nb6JDsh44OHbF24NVFE4BGTSDMtqNmEGipUnjaAH9Q/NR96zIZjGxlAcAuTpH\nyULZRLLujbsuNbZSystQJEW+EtZjzFmZvCuv9SluVefmxI2lFBXHvrRUjmjW\nDUkvPw1LZlIzlgakFufFYr69fjicX1dD6T1bHInvF2S0ws99Ix+e1ABVJHN1\n75ZIl5ou5ZbdAqnoPKSCF+Rz6gQVIffTkeiqU0N0dRMqhAj/19J70u6QQlVh\nF9f3asRJjCYcWUqmjZeLxXcijm4BQ1ZMj7kR2iPimm84dwO634KmsODUPs9h\n822RK6UvopM2AbyySBDdbEf4OQ0dGsc2vW6wkTMgvi8WXNF9I859d7mEfHlm\nACq3UtrNiDPL44IcYYu6Gtx/Ja28dF0xspM2i39ZCVn8s1iK07DZ7oVJBvct\ndNMZfPMOvkyxF91aADtXFY6Bc0QIVp8KMZSbvIH3pBtbtovW3SSSKOXNEnwP\nNAD+OjVEchqsM3Oak9RywCppRejOLAXrGQjE4ncTjsRo/86hHXARr5PsPeo/\n+T9TekdRT+YQIfdJTHdbMyBgjvMsDLpI8V5sWrYG548mPAYb3CUVULB2JdQH\n64tr5U/jPWAicpT02kgd/O0tf00rvblyWqT8evAPiTX4hIK0as7wXDn/eOTe\nNFVO\r\n=hQsY\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"f06f4b85314cc056d9f706d24386af17ac3ced28","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.34.4-danger.f06f4b853.6+f06f4b853","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.34.4-danger.f06f4b853.6_1619716660037_0.5774818032535864","host":"s3://npm-registry-packages"}},"0.35.0-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.35.0-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.0-canary.3","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6113e7f7da82dbe639905a56c4641653451f8ee9","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.0-canary.3.tgz","fileCount":26,"integrity":"sha512-3Nw0U1SNt4LQY3VLwyJgdn4JF8rED+0SvDeZmF/GRp74Qp4XVOh0DWtWBW32RtjB1FBDcH4Aqu27ZZxgRHAh/g==","signatures":[{"sig":"MEQCIC3l7iiYRCWu9wW4akAiqdjN3xp5lxSMPHErDU+1VuTVAiBqgrO7lMlyrlfqEFBKIegLly/PYx2PgOPmkxZ4p+fKWA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100307,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgizWwCRA9TVsSAnZWagAAMpYP/AiyNA8XL7lJmjSwAZLs\nlGLny0+rq/7A2tMYnJ2+sRZWsDB0DOIK/PTydxf7Ne2FY7a28XGXM7RInC9q\nKC0chnOH0yCVhRh87pfztRE/ZssKd+iRLEcvet1oKCzvvfxpNejKjuP6JqEG\ntKHfJSAQNPjmrrxF9nGJ+FZq9LSSel2TKpXksVn5As/IGRoN3o5mM3Vr2447\net77T0xS/P52atT6vUeSzUXjfTxs3J7iouCWNI2GBmYFtW/B350WrqZivfxK\nNygo/7VxgxeMRlZ8uKN0cI7RrwW1Rzss2rirmJOtE0+59w3VWEtOQ13EGk/X\nYjxnaTUi05YixC39LRTcqLXicA8b8bSabMSIE4+gYzAFJ92Ls57UM1YqSMbO\nqLpVaku6weoOKVf1iwLPjIa6vL9r/6oyjDy0pIw0UkAz6i9lnW+7nrw6qSgL\nk522EyyKRTqzLZxc2wwCt6aR9XLmih0E2u4XvdC9oEvKtQZOnlXaeLedEg9I\n6nQuCwRT7pfN4BlK4dEY8UdiIRowol2HGnqenHspA5Dl9/XsAd/a3scuS59I\nlRHQ9s7rfz12X6czdLC109LgBE0cUj5EvBAALgIR+fL6gggzEHkrfdTH0Rco\nHxi/B9ggFZpZSmlxWjMem3Aw+pxkRFo7TV4tzyvun2rJ3bUEUDoNR1X1LF0G\nU6zb\r\n=gGEo\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"ea2137b928bff7288208c4a36d712f09954a8c8d","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.0-canary.3","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.0-canary.3_1619735984548_0.20929516166081163","host":"s3://npm-registry-packages"}},"0.35.0-danger.91b725cb3.3":{"name":"@blitzjs/file-pipeline","version":"0.35.0-danger.91b725cb3.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.0-danger.91b725cb3.3","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"b3149e6059699af1bc7e36cc93257e49296054f1","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.0-danger.91b725cb3.3.tgz","fileCount":26,"integrity":"sha512-DJ5r/rLi02J/pV3il/elQiSimWqPcuUJF50o+8tZwR7ck4wSSOOXtd+xSODcf4Baswrk20ioL4xgu/w0C2UXdA==","signatures":[{"sig":"MEQCIGOBPijMKRlJX4UStqrB3p1C7fg+q8b7HZyjYhGybRp2AiAdOqJyuG25oYKWj3hW2GockR+rWsPbdbxjzczTLBgyMg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgkHLRCRA9TVsSAnZWagAAiHIP/1FaQKHZWL/GT+2GWyb4\nu/YJr8ij2BYV3e3b5Hvslq767L8tOZL1WWtV4/H88rYdxjfpbsM509/ofqMA\nC1sSVobMrHSm8YbkoVq7CFUEVOAOgt+DC7z4cvUu20iwH1jS6TRIq1JqSnpB\ny4fg/CBoKDl9qshtZfDoQEH/VHgMkhzwkwv8S/t07aYU3a6LNYaZnv9gSfhi\nn0ELER9kzUU9ag+f16jbVugCLl6CRvAzR/cTSDdGbruUS17DZYjZHprkMtK+\n2bu6G4W1mCcvmDOY8KTyN8Aja4frr3FUU/TTS9c3xLqlSbZo2dVLZu2oUZ7Z\nl5IJjzTJrYgo3pFkNmtrdVrAjMZD3J5iUcKV2IvCVJ/T1XgsKxVsjsyBYB2N\nmBvGjFvVDq8I/PVU47isgFYqoYdGHn/7Ycz0C6gNdsEgNbQb7LvQxCnRIBcy\nzi5rAT32rQPAztXiFTc2sKX6kRZgmMELNOvHuNrzLTjX9qwJ69pfKROHpJYo\neiDdoGfZodH+prEpGLEEVckjMUSvNUtNTpwDeHnGo/l3kxxjY1gFevtDD7Ay\n2SsXNpEz1tBTIo/0XUsLxyncPJiSLP2eJ/QZ5jMST/76vfba7xA0K1Vby/xj\n68Z7PhxgtW5MCWhnBmHJD4YuiXCNfND3Tqe6DsxEuIb50wpqTfI1Dd4NCDhp\nncne\r\n=Tp4B\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"91b725cb373351db8aca8c85aa0fd0b0d48bf510","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.0-danger.91b725cb3.3+91b725cb3","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.0-danger.91b725cb3.3_1620079312780_0.012769549812460257","host":"s3://npm-registry-packages"}},"0.35.0-danger.22054c4cf.3":{"name":"@blitzjs/file-pipeline","version":"0.35.0-danger.22054c4cf.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.0-danger.22054c4cf.3","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"278d46365691f65f54a8c3c3b40a8650291f5f31","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.0-danger.22054c4cf.3.tgz","fileCount":26,"integrity":"sha512-LG/6V1avUWdM4LYNrjjfWE3CtofZx/96xt2cp2NBEmWzqPmA4SEiB0eqmxgEwFaZgAuRXO+QJFD8MhuQvBwyMg==","signatures":[{"sig":"MEYCIQDUdjuiumWuStHeGh+I8nNiHa+jVxH0AYa0hPZU1C+oWwIhAPri4yh4klGFxmqfzwBeKuPktlag5C+t/LqM4QJTQKVZ","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgkHjkCRA9TVsSAnZWagAANB0P/1dBEerjju86m/GMb5nL\ncssz5qc5woBnaahffxVieZYPlBLue9OfiU5NcXujCynBmhf/On7gZbjRCbS9\n40hfFEPerENKuKpt7rfg7KtAms5UB2kumX7rrlA0TB+U0KLlntAl6juPilTj\nz+iLeiNehTsQ/C2wmvrNvyykY4cMw2adQNTzg9hYULh9FfbBIhJsWAfm4kbG\nfJmbYKORA7Q19XZzN5XYEvTsbCGgOmhjFXCqNEPYB5mBs2T2+zkxyEQSe8jT\nqS17JZXkUZWSA4rm5BK/XulGe8DfX1v2cQ6l7oifKJi74xImMUST8MHSyoNU\ny1qABrsCif3ZViO5vR5sMVF6+T9upA/VHC8M8m4/rAOTXlXmRK6arjW4iYhk\nAtXxMa1tn44ae683uJsBjSXq3zUGUnixUsnCnk7B+69OmPM6IJDLjxlLnEa3\nSnYMMpCNRNxknoZhL0IeXwGlaPJ+exjm3H3DM75zrJ2EyHCNiA2RfpRj9C8r\nJyMPyoZx3Nv027hRYA9at+NwcUNAFnDWA8sm5zwCwunXui5WQAJeaEVgzy4c\nwFzRdOZ1QUIzF1vzF7HInEM7MUWteoFvUMb/LRidWL1ewsd74QLt1MSKkcUf\nZweVqZyQ3H/KzRTjRzS6d9IWkLIiWRhckuJmfTTPqwDVGiPI1PUc4/9/wKkr\nCLtE\r\n=TIqW\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"22054c4cf0ce74ad8ddbba8c930ff578effbfd97","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.0-danger.22054c4cf.3+22054c4cf","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.0-danger.22054c4cf.3_1620080868217_0.4373291212197974","host":"s3://npm-registry-packages"}},"0.35.0-canary.4":{"name":"@blitzjs/file-pipeline","version":"0.35.0-canary.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.0-canary.4","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"42efe983047b950114da10194d5be42942e4c370","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.0-canary.4.tgz","fileCount":26,"integrity":"sha512-J09DmMPEPGLZk9Zqew3mLtgLVwtGfhdp9clTqSn1xPlIOvRW55BgeaX0XGFxFjpetGpS7shtMutX9T2mRNcO4A==","signatures":[{"sig":"MEQCIFvVpbCVxhE4Ly8Rg2usGof2YRKrgK+LeCyj792Fdma9AiBPiH8ZMdT/SbZjpGmQb3T4c2juAVaWitEvP7Gq2ZvUAQ==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100307,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgkbHACRA9TVsSAnZWagAAy2IQAIUb4VFM75qwIz8YYcNQ\nyUqEk1c4zYpnz9KHomIlLodQllYyJ64WjpAVBS4cO0Xu008IO6YcvT7nbBGv\nTYNmXDK4Ps85DmFX6gf2LKy9aPhGFCrkp9H4Cv/CeV8Xs8MtCZXpaw0IfGqP\nIHqOFvo4QSCrmvB7gjHCNxKmFViPQ2ygSDHVBItCwsRon9BjYqSIJi4NTiu6\nkqCv/kve6jPuE8wQJFlEbbXrREwCYQwFC6WZg0BalLU8GqLBtNWBcCDs08Rx\n0vmW7+snFfYPUs5FgCQttjiyKTBq8qiNIQKXyKCDTZ+juHh5wFE488RFczgS\nLm/F2lMyzUQ28PTit/pvLguuRD7sTzgeMn3esTeOUxBL53BuwTM5F9clAc6K\n73g2ICDL4BFCeF/xKCPjSSM1Yl8cakMsPyN7bwkxieGHTDIThainn2eHwGdG\nSEslhGCv37X0S7rquTupZ9VfnPWKI+5vaCotbQwFiyEGj3JmZzJAdo7nnNwq\nx8r5L1yUngMDLJDToMJbj164eORxXSROoiFMP57/ZyFKgC32W6XkeWT1bH/x\nespoTCyZBVoGFmQ4wTBZwB+XL8+R2qEjyuC/rqYeXTNhBlgWaeioT+wkeXlx\n4JRub1720xVLBJgbsTjcD7ud+XUmEW/o8ASGSwvKraX8MuC3Ol/lKzvpAAPg\nGzaK\r\n=Qzqt\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"6b995be962b800500109c39e3af7d117094f6b99","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.0-canary.4","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.0-canary.4_1620160959630_0.5773760797474816","host":"s3://npm-registry-packages"}},"0.35.0-canary.5":{"name":"@blitzjs/file-pipeline","version":"0.35.0-canary.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.0-canary.5","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"176ab9200e779201ab5e0faf7b43f1b0a32595c6","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.0-canary.5.tgz","fileCount":26,"integrity":"sha512-knIOS09PmdGmYHFgsx6MPq1urih3fG/cV+bxTah9Jv8Y2ocpeKD6fv6kzfm/K3bY2xiboPi3nB8p7XMNwntYBA==","signatures":[{"sig":"MEUCIBClzHiRl9tUXZlq6i32ylb+SBn22VOdsWCV6ryCDcv+AiEAigaFnDxhXuNDzBpnN7WxgOMdAUbNdnJJJdW8ZM7PDmo=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100307,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgkvYxCRA9TVsSAnZWagAA0+0P/iZjLhG7vKFzQyNzd8lm\nQQopBhbQrj2FJjm9BeInZL/Trgl1dQVf+wtYktsUDMEmwU5OLdx7SQHC0AGm\niYVh8dkkOfBMd+wbPoVOlNf30oxaPq6qeQroOwlcV38YYbCozIjiqWcsjwRS\n5wsVrg/MeCwM0PWCCFcIhwhT38dOISZADjRp9SEqZVny5z1ZYYc1dUIziBaq\nlQMGzpii8iVXmWP/rJ/pkPOIYHhc8tn9+GX9srI8NtupHt9UOMejwC7sdtk7\nX2NPdW+l8DCeNkvD36DCFjW889aZBjwXKeFI2uQWXN2/ulV3YrhHFGsrXXPS\nFs/DoPVNemL4CDsJigMaLzbMkQpjUJbqJsozuO/4kWvQrW3QIZTal0/5igGQ\nc9xFmdLUJly2Q+pll/AvUh5XGr6y2wnUhutTgvVeB41OUpldt8MfkR1RRYZ+\nOE6pAYQr41h49FBYrJ0gE+nyPnjBnowFiwwdZUuen5fRCNVf+k+2jfQvuCOI\nzBBwWkD9YG/TRAgbZDxf1d9gRZTgzcdNS65KXY3OCboLxV/ycuCDmpoUKW8/\nXm0dAKz8hLEFWGnwzHEtWeojJcpDiRLXsx1kp0PU5mZGsmldPFOV8DOgdbsZ\nLpHqQ+ggcsWM+/MWTBvPiqgu+I1mxVx1OuQhSjCgL7Hk6xZZishSbbZggAXi\n4IBZ\r\n=BU9J\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"162a0a48d2cf7a26badd59aaf08d22fb525f4513","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.0-canary.5","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.0-canary.5_1620244016742_0.1372624086526235","host":"s3://npm-registry-packages"}},"0.35.0-canary.6":{"name":"@blitzjs/file-pipeline","version":"0.35.0-canary.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.0-canary.6","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"62fa9319aa0dd267718210de5cf2e6696e0be145","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.0-canary.6.tgz","fileCount":26,"integrity":"sha512-841mUNGe9Bf6wJdMrQ/TZbiVBLZfpqgYxHzDsCoCurv2bv+u/iFvGAFc4UECCdjzN4Sfah8TlrqD1n1bJoB1tw==","signatures":[{"sig":"MEUCIQCzirst69PJqUA7c2OZ8MfvgT9MRJHGQ5I1gPXeAfUGKwIgPvuyJR+asthT/Mi9tRkJqvvzYJOUlW+uHkeQPGb0HVs=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100307,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgmbbQCRA9TVsSAnZWagAAergP/R89Oa8qK3cFFzzYkf4i\npk4ho0v3oz7S4hTBFgjSVVGuqcy9WL/O8NWG3HVa2Uzx4SUaJwwALyqNmH6n\nL/C0Xil/5XyJ7H7wliHlQBjphI/ujSOFQXyLZ65p+Ch8W1KWskz8k/Rflw7i\nqzG2H0luETETP72Ym+ro1w4mWGtl9AaYDhAHKKoZsO8k9AE4mJitqbJz1ssF\n6PDXrV+PyEcWA65R+B9XAxcmvKhBSdZomRTInUcKylmI/BmNQMjf0UKcBNaz\nORPr85W/sUR+MM6LUN9Gp3SOKxor3Jm+KmslzZCpEbEF+jPVz//jOYM2CIbh\n8HLpwao5cwEd153Xsv9KGmoIBKf1KkZ4AVO/ZK7ao9HXAFaolXai4hBKUki8\n/X2+cw2kXIbovGbxXJs2xiKHpMNf2JHwrFPdwy90tmZWi1bCfV3l+nsDa23O\nHm6p/QGygAw4l+EC25hOYKCZrcR9pAfKG6uEgfZ6aQWbk/2N9iFtzye8DvqA\n9dGEcKgmdmcgezZlalkVtdsmxCDsYgvhQcAZ9NsXCmPBqeWhN6p96n+7w/X9\nksHHOSKDle7xcncVA/OVkC5wqkdWn0YXlXOhNZebLOmdbSswQGaio1J9v/+Q\ngdAB5ZD5DWb/p9Oj6fT0ycRJVDz7jmPZpG8ds/KStixzrfIM/r44w1nN4QbR\ncLU5\r\n=jW6b\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"37f3c951d810428900aa4989989d78fb45a71081","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.0-canary.6","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.0-canary.6_1620686543620_0.8569378365389462","host":"s3://npm-registry-packages"}},"0.35.0":{"name":"@blitzjs/file-pipeline","version":"0.35.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c744e1ec8f873fe725162a10f8da2d9402efa721","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.0.tgz","fileCount":26,"integrity":"sha512-HlmVUJ8fyZsFsIm5/M2ENPORlfaR31RXolQxkeEPKwbWVK7PM0rQ6pORnDcpKTnwEjMhtykxIWGhRS2gKGdwyw==","signatures":[{"sig":"MEQCIGGq2ts6CyR4JFRQfDzpcFfi3jdPsds5x4wP3aMxcj64AiAaDgd2g/nFX9lucyIcjiRUOaJJ41Mm7TbhVAWnE5xOMA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgovJKCRA9TVsSAnZWagAAB6AQAIdS9zYXBIhJVMARDm5E\n/hwTXk1hcv0JjCOT1vF66myxkCaKUvgchLx+0xmUpie6DhxZZS4hlT5YA4co\n3Ng1On4dNUtuPDsGUWIbjv0HAZUjbWmnrt24EIcPHx77J+puUGF1A0VlLD4d\nkjML1mzwujsHyTnKZA2NZQnB66Q8GawqgzUL8Y999d8MGIrEKX1dQNrKVapG\njGCoQza0plcruujdvBWZt32ic6PPqugctHzhdTidrAO9gJHlFeCLF98IhA9E\nvQ+FY87DjatJj8AvoqMjjyMwefBOL6fB/Hv8SnfKBSwUF8GUF/nxUlid17WN\nbQRJePVypEQvQgsVL899KUI5uh7njBGdc/gG2WObI1QUAKQaSyoVKNTvMC0D\nTFfYtGXztJNO2oEvhKstGgrFAww7T0+jTTut/wrbmTsb0weDBfDRmZXLTH2p\nO1HqhJBFlCfXeCdfooidiT378IAvNBaytYurxqSnFftY0OXszFryu0i3xKQu\n1+n1GwVGDt9JbHxW/VMnqnshBBYDMwCvYwrg/9y8dtRZ9763WEn8Kovc5bAT\n6qv/T4Q3gkl43W8k6tgtP3qUnWTwtpYwgdbSf+lU7y3Gbw1rkPajmqYHHGNA\nLjYzV4ArhvCsh/np/5BXeZ9t4wsfut3BfAjRQv3nxCn6+ZCqMtsPk7Anh67F\nAj0r\r\n=HTxO\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"782a71026ec2183219f1b1359d67c87c631a9f8b","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v14.15.5+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"14.15.5","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.0_1621291594270_0.12718046234234426","host":"s3://npm-registry-packages"}},"0.35.1":{"name":"@blitzjs/file-pipeline","version":"0.35.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.1","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"d0ca190be8a9bc80317a0cba5e39c89248a5ab77","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.1.tgz","fileCount":26,"integrity":"sha512-H9/v0gJh/YRkVV4c5UM9GX4CSrTNdLeXAOGyi1IaOIebhUPES3K67H8kmlBvgoLXT78XElvs0xPfQUMCkvWOxw==","signatures":[{"sig":"MEYCIQCRwb6Nsu6sPKD9Qqpszce7prmNGzQ3Xmk2hPPTrJDmPwIhAN7IxjIzVJ+bRPa7e19mx+ctfiTq3/a3OcIlkvzWn9+n","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgrrpxCRA9TVsSAnZWagAARQ4QAJmGeEKv6Rp2q4tmhzhC\nD24I4IbCj2hbi4+WC+LgzZIpPXU9X2ov38S8RL1lOvYZFEvdUvYFzDod8xQC\nc3REhFbp95YPICmk1szTacL33I96Ku4V/OEgM2Xn2yFNu/GMA4xpCEaVVRK3\nuqMgAMKlT6A0krqkMM2RQ6wf9USzvC+dugqgD3KZGc+hfVFOK0NkELrVKhF5\n6Qd7tnB7RvZNG2WLI/zOO855GT13N41fe4rxRap1tK2cVJWNZ1W/t3zA18eX\nPlb4UoOvo5ivJxTaxmIilQwvVECD4TzPF5b4ICpfVQL7Gua4+ywh/fxmaWXB\n6YtNe3/aRnJBCgWryqlWEt/se6X6Kz8/RiI0E5Pwt8HKgnwnfQjUyYnn6QS6\nHYHPMbgYB/Qvf/rIbQyj6ckIf5/0UYfir+OnlT/0qinDlo4+Fbf/09nxKPRy\nLKLmqYSqfkGsjmfv0KvsIbF6JOMYMZhbOTQB/e1PkcuVNahT1HF8dcig//K3\nRngK4RDXEw+hbWxGyehEhhQ+CBF9QCOc8M9037VYhEt4Amn9j3TK4nEHBsdt\nQJNDU+9wyKndm39WAwxjckykGaa6n0VTVPBcT9wxNy9d5B38MBpeB/7lwgzA\nI/NDZjdUVCrsFl/C2v2RrWPzjLSRJX4IBhgHqpEKosdtB7EadN0F/LpQ928M\nEsE3\r\n=gteN\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"d86b885897c1ea416c0683882f48a9ccb2dc1c62","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.1_1622063729361_0.6252446004064636","host":"s3://npm-registry-packages"}},"0.35.2":{"name":"@blitzjs/file-pipeline","version":"0.35.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.35.2","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"085bd3969652c5b938cf29d40e2f84aa793bfb94","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.35.2.tgz","fileCount":26,"integrity":"sha512-D+4IBuCixNnZ6+YreauEw5VCM+ODeESUTSgROBMXc9sPyXFFePf9g5JEn5cUI7gHdL2QO7wnBbdcGNzisOK1DQ==","signatures":[{"sig":"MEUCIQDgSaSr14LJaeuhTlBWG7x+WzKFos8CFMCPQyvxTRb9mwIgBZNUNFoihUhNjvbLYnH8IjWCCC3V3/oBcQVR/Tb+c6Q=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgrr/yCRA9TVsSAnZWagAAfaYP/RjYEPzquDW4hlC8N7is\njQ0lYqC2mh0ViSl7k8xsgfydvHoEq0ipWuE5mr5UVgyyiRTpCK7mvNavu4KQ\nJayXQJBNBGqlDGN38Xti+PIjSVR7CIu+ybmQtec/1dLP3y56i84gE2cO0rx8\nJAjgxYrlcDY/e1VppI8tZWwmAiZ01CLKILBbjZHzUrncekZ1yqVT97h6q7lr\nYYCgygYIC3HWMLqtXZ6CDOpptCzxw43qHq6h8UEjiYpEq5H52D2dtYrbL4OE\njGCB4X8NuqE4LgH6/Hr+DR9MIx26zTUyTAm5F/40P6QJTOr5weIn35Q0Zt9E\nkdbzweLHH8AmYz8uAWflWVNHyB4U801qTRlKX+ZubhL+MG96JucGLaayAAyU\n2kPqEd1O22xyF1oNYI7d+TCXueXoIj4KOqwRiA+n/mJ9Eaf7drhBnv0Vwhpi\nzr2m0Hy5ZZsrqwiYncFwQvomm6T9Ra/wl9z8SR1DXppunHksqjTddOQw5mqb\n1unU2V+kNjUoe5wfnvM/MOcvZB7mWkFG5YsTcXz/vuc6W+BwDljWUkQqWU0Q\n0xTG1wA6EdlhfBY5TCiVOVUtjvDeFVYVXzoq+flNmhGQuhA45yRP3nwGWTpt\nP7eXp0jXEZyD7zXuijIn0UE6ZJxyt1WNOwvtVMUga2YIJbTeyymFs4M+QPbM\noSqA\r\n=/6Gx\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"eae332b0c797624642d8eabec4836ad4ee78e10d","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.35.2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.35.2_1622065138116_0.4348942797136319","host":"s3://npm-registry-packages"}},"0.36.2":{"name":"@blitzjs/file-pipeline","version":"0.36.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.36.2","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"a657c6544c169ae518c6ee4a9fc21f85d70e404d","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.36.2.tgz","fileCount":26,"integrity":"sha512-v4jfdcV/2KVNsPR6g0lVjUBaKdXLkTNbwZialJzllvspwCbi5NpVKll8DhngAaelrZWSkN54b7RehhXKH7sa0A==","signatures":[{"sig":"MEUCIDxOxoy3k1rjt/+LQI6zT27Vi4X7GkCE1vlxDIcQ8TouAiEAx5OTUbBjP4GK2KroYIQRnoC1I7zLBxOBIUjhg7OFOhQ=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgsV1XCRA9TVsSAnZWagAAOXUP/0eYjFzAYWgStjE6gS6O\nXz96F4AxfxrmpmEkSRUa70iHNDMT+EOkd/QOSibTBS8SOjz5AhmuEkZLMBnv\n4td49bPDQIsuvOWNvHMzFzKWVIQ9xXxPM3FzEiE7vvxiDGR4EOv5nkxH7nyf\nDsahv7Vjcy2oDqJwGTXzjr7HryaRoo0OISXsptVFo7eP9uADd0lQK/lnFzTU\nNL1d4E1jcyTTN0aDcN8PCNG78dWYHZGp1Ymjl+WHt3juZmk2jmZW3bfjZEPY\noxy54t5VLoTtpxltnKXC0g5q5FGfToeQY+OuybhxWWvAx6mjraaqcZ92M16j\nidW5HmA6phINaVWPEogezicaUdEmQ3za0Wg8eP+RX3KryVYdayAciW58GcLR\ndDBfbf2E1gepZr9A6PYy+jecnxRT8k0wMOJBg/SMZglbIxQ0G8/VfGx/Iumi\nmDX9pPz2cE5c8CnhBjpLuboERzN5ylkUOXNvt06+/Ez830d8iJoPktpqCQWp\nW3L3UCcmgmirpYdQ50/Qg1JO5lhTBcFXOx09EeeOO57n00rfkpJ5gaSiZUHv\nI+DMH2cvOggycADFu2JEq8RB1i7b1fNDdHVIQyzTOMUemyJ/9x2lTQ85JtS2\nsh1MmD/SBPZas2vZL/GKAqXjldxyygVM+xdt8fRrUyRVFAN1cdRatkqcNYtn\n5TCU\r\n=J04u\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"53bb153df801c9e1727f98afacd5027bc482195e","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.36.2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.36.2_1622236503319_0.9283290993893147","host":"s3://npm-registry-packages"}},"0.36.3-danger.3b2531081.3":{"name":"@blitzjs/file-pipeline","version":"0.36.3-danger.3b2531081.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.36.3-danger.3b2531081.3","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"3e9e6de50f7611066dfd5ef4418551bc0ca74778","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.36.3-danger.3b2531081.3.tgz","fileCount":26,"integrity":"sha512-SjLF9jvoAtCkyjZzjl1YgoUbgOqNROJIkIAKis3PIDpgjYlFCvTuvl+VyKj43jCXpnMnTIvd43REt5mP15BTEA==","signatures":[{"sig":"MEYCIQCcoQffdXGpSxjp1W78MRvpb3PFTAS7rzhOkryw4OuQ5QIhAMioH6FHb4gtx7Bk1JkbLYSfJg2Ewl8O5d9flNhzghkL","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgssHpCRA9TVsSAnZWagAAz5cQAIw0sJ7bS17pVSSxU44z\nIGu06yCKL3n9LQIRfTiQBT5lKnQEPlEFPuj91xnv1olTtP2C8ctKhBlIv+7f\nlszUkJh5yWqnTO85u4GhwkBSKZmgKuOgUmNA7x9yNPOkagfQKT92wvu+hFrR\nvSU5ta0srQASPqDy16bjBxzBW8q/nIt5OHnvzvPlZnbdDjdBiUdgCQb6izGB\ndXrceuyFgDq24RBwrjdD8aj9Rvc3dBhEut5vlNQq8/hbEpISBObF7BO6u7cx\nS1fMolf6sHD6Pdg/KWwZoIlldwd/zRgI4zmFYL6NcSGsCsWv6rAytddU4G9Z\n6joG7QUe5UdbnArO4hQkVsxLzKk6bl+dbfWBvZUDWCXCrD/dOhfTqO+MqbK/\nV0QlIeepp+Q2G0L7G/CL5QCvTIaC+sRy7n4kOfwv5LOsGOTLQbZzTyLk3Hjw\nfwl8/YXlEIiXwykMX6DA4CXgJYEW5zEksMYHI+ZelShxv18Ye9L3edjTVSkR\nz7h7pQjQ/hvn6ZtBK225J2VfxWlhRCQH2ICkstlwN551pgFxHQIU53W3F7kd\nYmfiqybRuhB8NZPqcVRvYUNWWzZA0vJN/xTrbvnQpfTxpih19ys+W7w3aMG+\nvlY8On+5WagCrF51D0riiNa/Rqo2+8RxGKgtjvbgk1qaOZN4yUzu/gR9qYnY\nAlrR\r\n=QJQV\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"3b2531081e1149a668fbdfdfa5268956d8ff3e6f","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.36.3-danger.3b2531081.3+3b2531081","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.36.3-danger.3b2531081.3_1622327784625_0.6956232230205197","host":"s3://npm-registry-packages"}},"0.36.3":{"name":"@blitzjs/file-pipeline","version":"0.36.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.36.3","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"791246f24e4fc60636ce9fa3935619bbc48918a7","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.36.3.tgz","fileCount":26,"integrity":"sha512-xGo42+6MWbirDBSO8YSpDY9nYynWX9EGJLdu97drhj83bYZjoB30BSqwXIgpYuhaZgDpztcmE/zzP4lOrLca2A==","signatures":[{"sig":"MEQCIDIJq3Szs9tqxRQH5EcZe9Bgof/V08z2iaGuzll3+ZfWAiBKgsDhNs79l156V2jVQKIdLQcZcvGF7uM3PzDR6WigOw==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgssSbCRA9TVsSAnZWagAAFesP/1arT0SkKJzy1+Ms9f7W\ngdjnvY3mavYz9rfCvo39/cnIdCMi14yDP6JhBPr/8NwLHEvq1CJsFrGb82a5\n8KhbcVflPB+2boW/XaSaVl84brLiDKIFXvN73KFWAVZg2aUDhYKksB6ebWA+\n4kqj1PuQZjO5guPOCUHgeLishQOGzgXQI7Q925rSIEHQVSXTP7/S0ZoTx5k0\nNjSmTK0gq9h4YLn0ySwToYGTu50RXt5doyhJBzO1Y5mZEQ6QsTqBg3VwQaHy\n7arbRNOuHX+xa+7Tt2zMKfnqMMRqaEt9E6y7zG0b94BUwVNtvxdhVdt8KBlZ\nPztGbXWR4ewR9VOdSCBRKa4TgAi6pLnAnFLpJAOamqOIoZyyOO7BNOM4dv8Q\nMKtCERFMLrpvA+d5RWWgk3vKtll7tJJ2sp82s1eM+wi2Uq5L8hJlxAGwBtHm\nzUN58D5UQxe0MgccyY2RL2CR8qXU8AE4D5bhrU4QvkZKEJXSZNh0NcUql0i0\nsgBluhm6FcnWLmZW6gC4WyNlzIUK7t/EXOnjWWsTMsMraO7UjgQBWs7JgWqx\nAj6xDAmHsAArweUXrwIw5fLujYi8V28by5yeSHJIbatwWFDpNMGXmyTLr5L/\nPXuR3RNYpZktOpgk3on57gjeH2hb+zoK/K6V8rvqqMmUj2peNWXW9wGvAWe/\nh5A8\r\n=mnNa\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"a3cf482e8f57b4d9e67e1b99a66280ce96856d71","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.36.3","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.36.3_1622328474817_0.7982011170692558","host":"s3://npm-registry-packages"}},"0.36.4":{"name":"@blitzjs/file-pipeline","version":"0.36.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.36.4","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"a267ea61ee1e465645765957093ed698c96d254f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.36.4.tgz","fileCount":26,"integrity":"sha512-XEd4uHg2UZewedvQusJDsy23iJf9BVsa+Uub+o5ukM3fl+qqkVRhRiWsCXDj4ZGVm/V8zA/uD8b5gUVqerORGw==","signatures":[{"sig":"MEUCIQDf34nQA9K9icZx4esg/akpZqll/kutoQ6qisIOTVwoLQIgJ5EIq7VOAGzD3ZdOMZAAyCe4rQeQzhq5B40K9qganG8=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgt746CRA9TVsSAnZWagAAaqIP+gLg4eGOc9ozafulUtLr\njOcDeyM+MTTlMVOaMA7FIqZIWVBCDho7H2sA/Q9LqQdKOZ5S0URYJ9ec7IR8\nzlADsT2Gy7foOpnuxovcRyFEArIK5YrNJ27xJOuJZlHSTje2OK1UOIE0KXD5\nzie2UEy4pdR2kr69Q/7peS3sr33Id9hnLy40FiBM06fPaJh1s96ICOq6LJFj\nghEL0o80hSWeowNeXfqxah2QYt9WjU7WfTmVDVLexKHnCJok04gRohdiab0e\n1oyU4DUphAZD/LIq8yKjZSD1eAmjU/Bag/cLZNWZjGF6oQjFrGZlFb7LrBMk\nB8YVdCfxEVzkg6tSkYBxXKeKjVQlYfLZciQNF8teGa/6XPCRZuWeYBVeA8tQ\nCb0lHAE/dgWw1gZo8kgEmEKBMIrBmapDbdGt19UbPiqJVwkBH+zvwi0BKNTy\nTe6Cfv+SQB09a9FRL3QOLeC1FN5ViaFk+u6A6iwwpWR3g/qpqp47v701DVUs\nwWl6TFC7pXd0JNwHK2fRG+uLCgfpXeWsv7N5I/GzJGwRu/wcvxOaLQ4KMQxE\nRwuhJmd/d0a2sAGq754m+YT0EM4eycEfXikj4fvPAeIXu+ILNEWEY7K1zgBj\nYJDFkz5O3OZp6NDn6p/zHdtK0An9Ioye4nYC+09wn/HZQRJliKCd1CAawC5m\n65z0\r\n=i4Dy\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"59272f7334d5ab6279b2bba18ff9452e7941315d","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.36.4","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.36.4_1622654521872_0.209347900730666","host":"s3://npm-registry-packages"}},"0.36.5-danger.7bda601cd.11":{"name":"@blitzjs/file-pipeline","version":"0.36.5-danger.7bda601cd.11","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.36.5-danger.7bda601cd.11","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"2aab8e9fd9627869bfe209e4bb976cc874575a6c","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.36.5-danger.7bda601cd.11.tgz","fileCount":26,"integrity":"sha512-dP00TvJy78PHyZApA02k05dlPXr9VTEOchbV9MK3Pm/Scs+35AgOu8rJC4yYC9P3LOzgdGUHcnIkmpr1V5dbUg==","signatures":[{"sig":"MEYCIQD1ZntSYd/Uwpr0XdOkx8ku5723XelltZIgPQD1Vy1NzQIhAKGFEAO24MAK8uJq0AH+7e0mo6z5tcT1qLNRbjTepw6v","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100349,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgvozyCRA9TVsSAnZWagAAEg8P/0/Nu/4pyWpvhRJ5OwdY\nEyRut/mWva/gPKjepKKavPzvc+8RLjIm5yMXWv/zQh9qgwypMCNuKodW+5qx\nLtkUjFNpRD0N6nFbQ/gjUjs28517mRML+qq0aJmYI0lK+ewWlxmwnTzS4xFE\nqgPKRJV2O0D+/Y2LGgwNjFIvH7kMvWUyCBGpI2uxXfUD2SzPGQv1JuodTcUK\npLiXTFLgdc60z0wvj7rveixaELWvrFb8xu63K3b43vArFMJGhWP31Kn7QbZH\nom3uSCkXB78+FH4gPw94P+4LJfaya8Okl76pg9kB6t+njeQMMtedxZdPvboO\nbP2qB1Z1gIGYmNGSu7ZyMpmUMNM2+zWRk+KRobcb86OdpxyEQ4wRc/Qq4Qrw\nQhz0y1Yqdcp00UbPANoj1PUFeCFlAo3YPDE8TTtZRQ4W7avgsQJA3YfOa1Kz\nfopcBn26xM4rne83d5jzEGuuPnlcxoobk/JxEh7iTrdOviENDvsqn6QCAJ3I\njvnZsHDpWv8sJZoac09I3SKYvy5fJXUzE6+MZrlp3aXGbU7VBFNwpIOLHjdV\nHquv3hli+yxWGa73TRZUUwPnEqlPAppc67uLbbcZRia3vyCDKaO4T7m4Q6Eq\n14kU/QaFYgkWjf8Y1qBMP7jc0wYGt/41D56ZnSsc6g7XHSCXIP1ZCbedeFx1\nqoeD\r\n=+avq\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"7bda601cd765419b6ea15036c0f9f999ea3b7117","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.36.5-danger.7bda601cd.11+7bda601cd","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.36.5-danger.7bda601cd.11_1623100658346_0.7370986933204455","host":"s3://npm-registry-packages"}},"0.36.5-danger.e4504f3fa.12":{"name":"@blitzjs/file-pipeline","version":"0.36.5-danger.e4504f3fa.12","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.36.5-danger.e4504f3fa.12","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"47d0985a227326ba0364d5586d6884ac3382384a","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.36.5-danger.e4504f3fa.12.tgz","fileCount":26,"integrity":"sha512-Y8VqhNJB5cxkZOL4vo5pBrJGmqrC0p0zy2LfPQgV5CCfoKO2gpzCRbTLsgq9v/uRTmzKLfS4LVmxNxRnu4m8vA==","signatures":[{"sig":"MEYCIQCLZU13VFYk0Jf2vgskQCFixnKX/u8qMm7GeC2S1dJcIgIhALGxg+vxoyPIdbztIvtaXuxax/CVHiCm+YVAzKT8qBS4","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100349,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgvo4jCRA9TVsSAnZWagAA9ZAP/jPqCtQrgnLSh4kwTz2b\nagfMPW8ubTgmpKpLnabACetFow+GHkMy5AZZ/BWaKMaIEZy3BQ8kUmUqaHen\n+UR1Qffevbz6ZpuVGrwhCMU7ExvTWOEbKSv+WyMgmlyBhcjzK+ODiRn3A/iA\nG76RK2am+ok2VWjafXtEvn5NlyQsa8tf3121fvEHZWKhoeqXV63jmA5jRmM5\nU1K6sbto+XTbbOYTcl1D4vsaI5Hl09R70VbWfgpHUmVt9DDBVvwBCcCggYBQ\n6+OHygiMIO8VMNGP3cg/7ZC/xXmcjX3HjT/KDbragg4coSnfYKo7KjEuOo/C\n7eSnkO+YgecXrIoDbim3cANg4cosMzZ9Mt9+t8ZP6X7DvAOfzt5eReaO3OI8\nAGFryi5LL+hj+0kw32zwojjmk87WjOlWBoOlvKsV9rSdVJ7CE9KGNgeMrTi4\nuwp8Kv9q+m0ZoVk3dEx9ROhloCuNqEbth+RCyIve82s4Yz7mTDlkA2MZhNdr\nPEYCh2AwPF67aRLgjqhHoVZzxR0UWQitpctyea16mCsdOnZoRkNq+htusI5U\ngwI260vKLuECeuIPFPlGzRLRXm28Y48YKFrXk60bLFtcODFhGq2zKJjp+2LD\nEHDh32IHJVROnsM71Ib3zP0ezeZCM6jLOz6Og1CVqm+IZ5iMvq9jFT4lKz/V\nAEWa\r\n=A7gs\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"e4504f3fa3dcaf96bc930f7e9fa8ce80018c36c3","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.36.5-danger.e4504f3fa.12+e4504f3fa","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.36.5-danger.e4504f3fa.12_1623100963477_0.38124257768780145","host":"s3://npm-registry-packages"}},"0.36.5-danger.73705b625.14":{"name":"@blitzjs/file-pipeline","version":"0.36.5-danger.73705b625.14","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.36.5-danger.73705b625.14","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"656fd75620631ed451be2e37e05a11fb12df388b","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.36.5-danger.73705b625.14.tgz","fileCount":26,"integrity":"sha512-gWeNOpN7YAyYHdIEA0NoAZmZ0bZVU5LSyn8HWb4nswMBF0rgVBG6+IK1nKv5znMqJ4l2qgBLBHAnYP6SAr/cYQ==","signatures":[{"sig":"MEYCIQCqxJqN0yoMkhQp7znixQGYbuTV9D2XKRyOHy1M5/wAPwIhAKo2r0hNTl/yulXEaH38dGb++3yqUflXDszwCr5CvlPF","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100349,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgvprLCRA9TVsSAnZWagAAiCIP/RkJ3LPa3CWOYMonMqMh\nbEVDrwAf0RJFmjNQPxch6aJmSkUcH4/obwFdTBM+o/Y8fpoZqiqzl3S6w3IS\njELq4oopF7G9XLJH6c3e85bHl8122Q6jxJwQej2S4klbxd6Bt/As26YcWEQM\nLq2UDhuv9aZdKk6u7Thm+dcJzmQVWJyH1MN0nWqMOZYkQGVipnXvUUVgNaNF\nC2QR309B5lCBbQO0te1Z3rr4gvQVnAtG+qWKIqVfA5SWM2a39sxC0oBZTnQn\n8Wuy/mZEzIZtsBNSTuya4B8QjbyMqnjVtMn+NZmCXaXnSQnDF5fy0WSYrDQd\nwVsc8chVP/HAXoL4StlSuuJoAQhf72xc7FScx9LJODkdEKM0sTzfKyk7hGAd\nXbdLlOe8a8hfgMTQjSNO1Qh+Yv8is1/ihOX1FzIrXwVLWVcnEhHDvEfewkY3\nj2yUB6DlJP+/VD60BSHZzEQgLOVv1pWHKA/ZNA/Xq9bbMLq856FVA1smuaMB\nwkkcz3AtgdSj+ex8KVlBUHKTZDhWf9WkX55LeOR8Q7x0+rdLppZnTFpcUYIR\nBdZEOami3Et4mlX5I/7JiHx9fQ1aDLUQXB1xqEqRFmu4AU3Pv6IWPV8PHd1f\nMnod/1jmNnfRHyILf+0Nb3CeAajANG3uUPJYDXMJfv3NUM9nYYJn7pgkvfL3\nfFMa\r\n=qms9\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"73705b62599c29217d901e40aa4b23d9e663b35b","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.36.5-danger.73705b625.14+73705b625","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.36.5-danger.73705b625.14_1623104203128_0.47567378118816506","host":"s3://npm-registry-packages"}},"0.36.5-danger.5fb0f166b.15":{"name":"@blitzjs/file-pipeline","version":"0.36.5-danger.5fb0f166b.15","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.36.5-danger.5fb0f166b.15","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"56451feb261a99b404f3ed9d616ddffde66cb6d8","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.36.5-danger.5fb0f166b.15.tgz","fileCount":26,"integrity":"sha512-O8l7BeGM9JH1cM+I3oLTEihlZdi4G48+xTHEpd9dj/KEfdb52utQ6DL3ob95XUO9STe5PFGq8Fwz4PgCcZ2yOQ==","signatures":[{"sig":"MEUCIGr9T1sno/+8DQ7yRUuF8wSJEhwb5PzHZnf9Y0hRbwbbAiEA5/sMqh50BUVrBs4PmjxTNEjLv9wW9MZV5WnH+sMQfYc=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100349,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgvpwfCRA9TVsSAnZWagAACS4P/ROYiq2AoL1r9680249Z\nTYKCxNcQUl46YBQS90G5T7PtbuKAIm1KfVa84Zd7jBaD+kNhQBBAw6X4kbVT\nWQtfyIW8Fo7TRVzCxKJxJ/hLryrLZnZzcCtkC2FgQPuCwmYRkkJHBBr7fwfB\nQMKmEGKu7GVEZXXTZoI8b+Zoc0DEt7186KLtGWrG8ZeDnIJ62DYg52q/kQiv\n2p+fxRUwkOflIr2ThBJovJHfLpYCFYz8yU8gVcPouAQQp6JM9LjN+DBROwOB\ntQRTmPK91Kk7hvNA8AAjhUbTyNLsDeaAl1N8knsbNNIZdGaatdt463KgFKO4\nJUi5tje7whXPrGLN5cIAmt2RjAKjspm8V7f5yuPB9kODgvxhb9zgvohUHg/O\nm6WRLb9bPN2NilRdodvDR5ubcQLbOh3YHisUjEjAj9cqhpfnkllnKiB9055H\nPpFhCFQBl6yRaoiDjLlJE1bPKGH0YZjK9j8/1pYs/3va+soq1sAjDTiLJfoi\nMJveai7wAMP/tzvWqeVfI1UeZiH3hW3Dcvlzp3xYsjQ8IKNAzQwRwPx2Axoc\nLg93OWvPu+Qr4ZEKNWcTYAI3uSvv1UYk4qKAPwfFXMdt/ivV3/xArEQj/e4a\nr1kI9ceSkStxSRDOkK2dIeXdRrYEsLoM6LXzMsTG4bg7YpN7hsJlE2xgkql2\naf6N\r\n=VKrP\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"5fb0f166bcecf51eb981e0ea39b74e8b582aa4a8","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.36.5-danger.5fb0f166b.15+5fb0f166b","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.36.5-danger.5fb0f166b.15_1623104542959_0.17977796919920563","host":"s3://npm-registry-packages"}},"0.36.5-danger.e7acb2375.16":{"name":"@blitzjs/file-pipeline","version":"0.36.5-danger.e7acb2375.16","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.36.5-danger.e7acb2375.16","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"0737e1a2e10d2763621b70df6b568dfc5a517e1f","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.36.5-danger.e7acb2375.16.tgz","fileCount":26,"integrity":"sha512-i4pUoI2zmHCxH18yzLMyyD4wLroBevkGScBg7KAUqYmLU9ubn75IxeCLlqQSC2dMXu9d1DfVBEQEJ6Svhq68Ig==","signatures":[{"sig":"MEYCIQCjhuhCJM5FbXVLOcXunEC0hPlpk8Y6xKCLFuKpK49VUAIhAIz1cr3EuAgQOGBmvToiZ/YuL9EHc89VLyd8WxyqN4QA","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100349,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgvqBhCRA9TVsSAnZWagAA/QIQAJmIaeV0u+Yygrt/Omqx\nP+6lNwNuRUxBJTc0lP2XuvBrwhKlP91xvS2FwDfsoTQtwMUx9i3oxBgpcqhS\nIRClT4WgjIMredX4t1zUmQyRNksYvMO6MGYymmUvdTOWiioDiRV/g9DpWxSo\n+rv/YewK7z6TuS7gfbdIESmPM2cQWegXTfWWtyUyUM8pJgnQ0ahAbtL1GaWV\ncEHCA2cQRmh6ON1wYco2ah+GI9MszKyKEB0G2LidY+6myjoidnsYmsEtLqH7\nW04figPqgbQu6eXJrV5AYgNkpzmsvbWPVNgbYzaMY9GG4iSjW0ZWCOG5fr3M\npWTIfyoXfnRKhMJGeTmA4dvxPM7qkEOaIKDpibYhJDP9A9sH+yOXNoKMCRcl\nAAji7uE7WBn1PDvJYhcusMZUwcskkvM7oBAkskMNEZQE2afR27wN3sCx5oNr\nuLNcn/qj4tV1kkoF8srz2KDrooB2EiSJkc4kT7hEW4/NflCD/EBSUTqF2rKO\n49FNgFFEOSw0MDAoadV6XsvNsgPMOAqjhVv1Y6WnvGyKbJKezD4dl3E7Z8uN\nU+j0rGJaZ0FviedSAk3AhkbZbqcW7JnnESf4q4NwCsAvGxGZgaChkytvM+Nl\nKQOjQeG6i4kJd/0ViB0Uua441Ie/CNJfH+s4HD56EpvD7uyQQzZK2zdJTRvA\nUvpf\r\n=xgda\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"e7acb2375bd85f5c951312761d25135ba8dde1eb","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.36.5-danger.e7acb2375.16+e7acb2375","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.36.5-danger.e7acb2375.16_1623105633686_0.31155457823330335","host":"s3://npm-registry-packages"}},"0.37.0":{"name":"@blitzjs/file-pipeline","version":"0.37.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.37.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"2c0545d68e1054096183cd610dcf0a14c77c3d8e","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.37.0.tgz","fileCount":26,"integrity":"sha512-8ibNgXIMN45vKVoa4mAf/+ttFMpguOaAE6KFTY1QER8pxBytgZ0q+isG+zoIWjp+yhGnv1OFD0NcoZwE/WlxMw==","signatures":[{"sig":"MEYCIQDq6KB4P3TWzq+/+qWSFdxVDhluvVW/miB9CEN7jIGRwwIhANw2saGHBKFXm0+kKJIOo1LSAuXoXkwcmyeWk2oAB14J","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgv7ecCRA9TVsSAnZWagAAvrMQAJLZKRMjOUP1WNjq9Oho\nHtVQ1z2Wv3/U6i8svU2ySg/PSI+hh1MCy5EBQ9QlZOqHZPpGrYodObyZ/HOn\n+D7KRigGmqz4532YPb+rubJ+pz4CGnO0tcJrfeOafvlr42PZ2vOPOB81U5Kk\nnMr8P6T5xoK6FCvRfaSVyKNJmQJvM05hi776WJS43jBBknWjAhNnSNum00lQ\n4+GaKJV6FqXK1r8RPt1pAkQZiXDMySTG/VAsTSfSi49aAFHw3rFkkzwPuyQC\nXOi0cYLSrrX5cAhL1K6fLvpgiL0HGPMt9/WUbUemf4jF8z6gdtq95lQwOzT9\nO8yWhChHx5b8D68Z6ulI/g5XhbZyEMieDrkxCkpC5wtFdX5rJa6Fo40ww0u0\nVqdywI0LAN0tUyYiwgJigDJ10DL+jfO25Ut2HvKF09o1OKRS/ruaGgdRmg7N\nHr2Kl7/+UUBU4mh4ItrsUmlo9gJYddWRGaWLCgzfbxniXkxWKVIKxlpiuPaN\nR7Wh/94RtSIUT3FRJ3tGTp9S9vu7yBlVPvkpLT8+lGGU8c0S5e6G9rmhSk32\nlHtDKuUWd7lcd2tAgtKgnO0SPWdV23YJtY8+HFaFc+Pnn2eh2qjgFEGc2FDi\nBlVYdIWB+2lcC3/dtpBquZ8p41YMToOwWJdMASUMiC9JoGabDZ1YXqlhVHQP\nC4C3\r\n=VPRH\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"7d5f437c7b7867450430aac85ddf6f05ceebcfb3","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.37.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.37.0_1623177116084_0.8509161392095317","host":"s3://npm-registry-packages"}},"0.37.1-danger.0c154426a.8":{"name":"@blitzjs/file-pipeline","version":"0.37.1-danger.0c154426a.8","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.37.1-danger.0c154426a.8","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"26fb209295aed4ea4b39e589c4b7155163622660","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.37.1-danger.0c154426a.8.tgz","fileCount":26,"integrity":"sha512-q5qGRVcW27zPjAGbo1UNWGd7HWfLFr+NHm3gusUYfayke75kP+CFCHs9i9+JwBtw0baSwwxoSLmNT59b6XCCMw==","signatures":[{"sig":"MEUCIQDDfYKInw1oxm+HI3xNa1BzNbRAsx0dVrNsHF64hb8sBAIgTWE79bsonWMzAZFL50rWzeo4vaL9/XEkO7kFKlmu6is=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgw5UxCRA9TVsSAnZWagAASpIQAJaPSo0Ve5fARWadMdZO\ng3zvqkz1eJjqmKZZv/eN8gSZNxfQ3zKlxkQYmqJlJ07aVjEy3BVieKemkFI2\nI1N0gTjMxewC/PytW3L8bOKCRaFo3UYLdlEid+52jc7UMWMAV5UaUI4GpegF\nFEnVU1XiWFDI/kABEkGBUHBUSeh/jDeidJVxpQBBKhjOwoUZqUeEK7bObQuu\ntu0cQtkRySFkaxTDRiUc0F38tyXddHUVLYVr/C4eCNLNfdK99KuvBJJLgBKH\nN7rEf4GWRV1iybMxVruq+kuuCPi75Pmw4qnTldb2SMbUyR8IJQhdWOY+s6az\nj+3+ElfX7b3w2QxuRg9Wg6D+yMob00rdhe5wxuMOxvgvhvydaV/TZAwVg4sG\nKSGiKhwqJl095nw+xwmbm56Ags0CvtgGAq0rkiwXVIOXviC9UeDD0SqSXcTZ\nIcckGcqNRB45XG4gxsqGSumx5eMb6Jnld9x2Pl1p5R1204wHVk4pnqcqshdi\nFurpshiWGaJUZjo8k8yA6FKZ8uOos7x9tkslZ77b8ZD7z+njbRIuX6deiWxF\nMFKm//sR0JniB+p9W7pHFs+EFZc3DghCbjYg8ZxAdVUxPHxrFLQb9zxF0Z3d\nU1K1Ob9HMtFl67s3PGm5V5T35oR+Ye3e9jCIuv20a5/5J+xC56px2LGiISrn\nmBo/\r\n=9YUB\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"0c154426a3c891e67ad4847e142cfb3dadaa2e42","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.37.1-danger.0c154426a.8+0c154426a","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.37.1-danger.0c154426a.8_1623430449698_0.39646602718014123","host":"s3://npm-registry-packages"}},"0.37.1-danger.e2b6dd277.30":{"name":"@blitzjs/file-pipeline","version":"0.37.1-danger.e2b6dd277.30","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.37.1-danger.e2b6dd277.30","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6a15de9b7e911394a0dc9a9c1256c6660db8ca58","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.37.1-danger.e2b6dd277.30.tgz","fileCount":26,"integrity":"sha512-WwgmNv2+l4VLM/ZGsWezkBdrNmdNxoalQUjRk6Shxk+tbHEIasWhDBddfW2nxq3Y6k/VbkaMfl1/nBXNYUJe3A==","signatures":[{"sig":"MEQCIBc3SQQuOYsGMAxsPoAzvBJymrqTfinDdjN1ewHeR00RAiAV0QyApiMbpNpryFJkGrJT8dksjf2CyurWzF4zz0F+ow==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100349,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgyl1uCRA9TVsSAnZWagAAOM8P+QDv7CvZKgv3gd41OzDw\nYgOB/t7R8KbKUJwGpswzLQJ0hk3GNC0H5yBibkx2ly8HJzF0xv7THtDN1g02\nrPCToC2KnpyDbDL9ofO+JG5ByEC3Pi0AHl37uWTHybkE7eksovAK7KsFTXKM\nyNZ5wNYMaS/eIHSLiAx98eW7T9pI7AWaetaxhKgqawH70ahPXffgWGYJBC8W\nhIpGVPmXWeWJaJUAiDtPCiQNr4SlYEt8v6VjCccIH06eaz/gXUB4mh58EtZ/\nbztVzWGNJSxazZZPhe1bR56XWT8cUvASb0F7gjCYNPXoptfsqWsF54gkUQ7J\nBEKTVEwlqRVpHc82LkUOabRDJmLNccR2foc0o5FOShdMBLN/zfLy4OR3wJ6h\n/mjquf2CyxsnnPH7LhmrEBy2aRYReOkO5G8xD8N7xjao3Kn41Irj6JONb3wV\nOj4ly01CVdfozwSjc/M+3+lA+jaI5YptRyNAPrWFOECduLM4kZQExzPwXBEl\nBD7huCI6/8vD40gWTo8Nbuujm1AkOkxV+dUY2CNfXaUV20Mgc4gKO4RM1NBB\nbeEiWEX3GJWB3LexTX8FSTrIAkb0fICf4n5jv0inXQDq7qLzNA8uMPuKATNm\n2Oo6g0cACWgmGaJQIiRgycfkPgQEk+0T4YRXKfd9Dx0IgCe5260kjOl6eDkG\n4Ert\r\n=EfEL\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"e2b6dd2779eb1a00f18832347c8e67d78a773b15","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.37.1-danger.e2b6dd277.30+e2b6dd277","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.37.1-danger.e2b6dd277.30_1623874926521_0.7302252543853491","host":"s3://npm-registry-packages"}},"0.37.1-danger.7c1b90c56.33":{"name":"@blitzjs/file-pipeline","version":"0.37.1-danger.7c1b90c56.33","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.37.1-danger.7c1b90c56.33","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"5fd444bedf4776586ca7b4e4d88f5beeb3641595","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.37.1-danger.7c1b90c56.33.tgz","fileCount":26,"integrity":"sha512-Q4T6/d/tpNBg1KWklolPbi6SqPkbZ4z3s7t/R0AXf4Qln0E7lZhYiNs6peMT5o/dxMbshMZUUBSs5/pVdeerNQ==","signatures":[{"sig":"MEUCIAC+D05PJ7tDfj4OsKrOAQJAXth/qbo0fFki2LvtLcHXAiEA3ANK6c89QPbWupBJB2eMwk6QcGSkf+VRz71rZuof6Dw=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100349,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgyna1CRA9TVsSAnZWagAA/t0P/2N0iL5C9vEzIdTQ0AyL\nJIYFlOCajNKcsyvXsDv9B4ThbUafT5qv6M6MlC2hnVwl/FaghfhtvuocUB6D\nnTDtSNbCb8HSXuAkyM8InfLY4Oj0YqXt/ZGL9OcPHePDVcd++1qcbKlV5BAQ\nSr1P+TMZpns9IRQX/LpLH771igBn6qaIRn4OYK4YxChxLMMIEuWWg56kPXg+\nzG6/T8Jf6E5SQ7nyQ0Kr34eJdFucUnPRiNRZg17fKSO5hvZPFvoemAfKj3mg\n+e2Cg3bGyosp5k4kxr1PaB+AdXz0MLnqKZ2woa9h9SDs13JarbpSzAht3AM1\neTo0BqdTnUmbPmuXD+QVnYeMwNSKec5PgjazwMmpxCB3yLX2whI6sd5upCoL\nhMVL2Sb04/YXQWpP8m4ogCbR3JG9NZLrZ9bBfYTaSx/LAC/VWiVmQGEWCvKG\ncyGBPXndOfaQV5JV00P0tEvRz8EX2SLxSpfj3VT6cDjoWG66/uFUjXz7aL95\n98vgoKLbKq/T4J6DAaCYYeSmkqHLNMrul4nBZcBk8/IW9t8tayEayY3JfuQY\nU50iM15AOzpJLJU/QAcAhbTKfxul/648Sj+KGKHT/xu/lzMzzpafsBdQG2Ic\nrgJ0YJvOD9oKQ0Cj9otWwjRUPuVcDPTXRYR+JbQU3sK3VbU3664sziJOhD/I\niUx8\r\n=jCSx\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"7c1b90c56f30dc5cf136275d2f587352c5761f23","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.37.1-danger.7c1b90c56.33+7c1b90c56","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.37.1-danger.7c1b90c56.33_1623881397165_0.19437937126667149","host":"s3://npm-registry-packages"}},"0.37.1-danger.a640b1112.35":{"name":"@blitzjs/file-pipeline","version":"0.37.1-danger.a640b1112.35","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.37.1-danger.a640b1112.35","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"3b182174e56e37fa1241d7736d37e697aceda1f1","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.37.1-danger.a640b1112.35.tgz","fileCount":26,"integrity":"sha512-Bel6XX1qoeSX6Rzp8sdVNujarepXG/L+vFRVkYxlL99CitNgSWwBvzLaDKB75NMce3dN/uXIkwyRWpCbi/KkYA==","signatures":[{"sig":"MEYCIQCy+LCqrf8ZJVQlQlVMZwdQU3k1nah/iCyGwdMvsFz91wIhAIFVkswdFmhWjXGyArrqcLrmtojCbovo5dQF2XnlBzd4","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100349,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgyosxCRA9TVsSAnZWagAAhfsP/0aC1IqxxioImsCKt3ef\ndYQJdcl1I4VzhyIpBfuESRtbP16WIy8lh4KpfiHg3Sl5AqU9VxK8hbT2anWs\nCBiUEL4dqSHqm2r41GPEnak3apWPXciCWBC0fG1PTFIsRlYpy3HYuv7RFvRB\n9FBaDiSgMCQXz/leQ9NzCTsJun9UZt26m/RW6jVne+Z12u3E0LwYftY9ppSm\niJ78poKZ3S2B5NxaA/kaBZWFGH0JROJm3FLwjZZBchM6nay7ussKADUrqU/F\n5DPP2SIy3WGqJ15/bexhh3alEYj84K5DU1ZEhHaSUBRnOKCfysxalNAexGTl\n8ZNzsjUWZ8z/pyGsKuEw6/1xOSfaqdG6Hv2tlBrrU62jJ8umEFdlH0lRmHv8\n2VFDH9mAfvgDzsA2WFluIpjAAmW+SbbZ5/QBo2NNZ7b31Q8yEF4yX/nFiTgc\nWL+0/txk0THy5thhjiO4UmxMVZtUw3+xqPOPLSiX+q7b9mQuWju/yZRthq5I\nJW0XHC/Zbv2ahIiaiyrnBkk3NEu/FuEsnchqSc7nkSWOdMzJ1s29RVhL6L7P\nbJKRP1B+IKe40N10qAmjuXt/KEXpNPpn6nzN5fMwHToJcW++UVhq48hS65uB\ntaaF+Is638n4dZWf9rE97OoR2AoNwvTtczP+kHEFicEoXmJgsSfZg20A3p5M\n8ll/\r\n=1HQm\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"a640b1112fb696cf52c6e864a71aec5fdcf7c933","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.37.1-danger.a640b1112.35+a640b1112","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.37.1-danger.a640b1112.35_1623886641383_0.34111598472225246","host":"s3://npm-registry-packages"}},"0.38.0":{"name":"@blitzjs/file-pipeline","version":"0.38.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"52252fc642fa5c01f15910d018f7452a3a1b2ed9","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.0.tgz","fileCount":26,"integrity":"sha512-25mvrbZFKnZoMLZq1Lq2HCphak8eWT2W/xx7ec3KjukUeP1JmAQvXC0EJTdSgS1ntdnHBUUeOz5ZemOv00klvw==","signatures":[{"sig":"MEQCIDIyArxGjfjnXGZjszRgMS/QVgMwiZxYvcHcnl5jx51vAiB5TEdets1ef0ZPXUv7BBHw1gC2CLokIFiADQc2y8q4VA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgy06eCRA9TVsSAnZWagAABjMP/1XDrwTOvV1cGlGIP8DC\nLPHKI1XUTJfzFlERR/8OKM57CfI4pnDD42C/auice6/ePF7kLimPiOnzbK7R\nxarn+30L/RqOv/z+bjjJveh67BbLx651yDUjjOvJJKMYfgNsIkrJ9eIdgeBA\n/vlMnKSf0TdiscvSATrMSQNhcAt+DbaRA6lHj/l6pP9NUC9GKMY6w4vt+NMx\nLyD5CxnyzGE4IS27Wy8xPwTNHLNF45B5vraP2d1GYofIVAsj3O6UFpwO/3KE\ncxwUsvUcyLCcBzdJPS94wN8eMyV/2XPLEbF7eNcPZAcIjlQGKAChB1jFcz2X\nVj7obzMEZ8ddQ8jOM56p0GQ1J62oDbSMg95Uv1h4jfzhpq8dXdjpVfDO7Ejh\ni2By/PM3gsmaliFgbSy8unEbNbC9PHlo1oOMG0TnLmtA2FZ/dFdfkEWawT/c\nOzNoRxKKYY8arf8PuQ/eTXtb4bK/E0QVNS+vUHC5T3O+9UuiGA+aZVMRhckO\nbnadN6DQkEIqVsV5eNm2EAV0GwoVkR37mHCMZeRCtgU8jCCh31ZXwalqkcxT\nY5ntg7gQwL8AfKNmoV5NKq7MWXZ7JO8QEv7SCTgCkIgkJ4/DOVWtGmpqoh3O\n2nApHody1xskjg7lAgU+wZC3LQDGza4nTs0w+InSo9Mi0ZbeqW4pJAGdFA9x\nZbmw\r\n=LbTc\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"2111ca6fae55e3ef909bb64f258b42ba6fe9968d","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.0_1623936669734_0.07417823733531015","host":"s3://npm-registry-packages"}},"0.38.1":{"name":"@blitzjs/file-pipeline","version":"0.38.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.1","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6c4746aacf207cf062118bb7b3e5dc688dd2da15","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.1.tgz","fileCount":26,"integrity":"sha512-IN2WXnYdVzsNbWVTZAdCKYCOliB8kTID46S1U5MnP1/K1Q53bAvxUlVWyFLoJTo4MNgjlTdn6PEbjOkwHhFcEQ==","signatures":[{"sig":"MEYCIQDcKyZfpqGKaiPIjujrqEs2D+z+qKDyEBfHRtkNQ6IMhwIhAJUpMD1aDlsYfAOnwwMlD2sFfUQgzc0EkYB8rEmY8ICQ","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgzn0jCRA9TVsSAnZWagAA/WAP/2N3LyWIHAlYRkLe6Uyt\nUplkDm92yTfXVB/pQl3q6EskbvsJO+fogSQ6AvUhJktmexwGZOkWBJN260qN\nJnEF8lW55LrmWfCxpQVeOVqZrYPfyuXeXlgynzFIchmsdL1NdbagjjqO7h7o\nJ9MLFfz6Eh8jSKMNwmFMyMG7t82wfjJWD7hKq4bhsqM2khp/e/aU7GbF4VG9\nxiRcdZEeFZ0Ag/pUCkpJ0CcCbLR+fuKRehtcCCx6i2EAIloxT/nc6UUdswwL\nTyj7XF4g/MozWElbLJOAH/acdbBJmgkqLP3bcEBVnSSX81bDkGj3BQG8LHD/\nnBZrqPZGN5vZRk2/lRszhiSx+CyNONZgO5ITYbiVJ4/MmKQIfidZdWy4QU8/\nx162bnBI4/pKm3wsNNebqHIxMwMHYUg7fHeJXXSW8Ud3VJv0NJflAfAsqx7m\nzoFRIOfNQr/TyBfMupXw212q/CQVAKQz1u+zDk9y3lyzF0aV8RH+Xr95Qll4\nKYrRMKqcP86B+Rgbp1GjFpF9jXsdy7FMar25x3zvPT83uPhbTKLxYBC5v8N6\nxWaNQizvSNGBjLPS6Pul4lnAW/JawoX+nFTp/OI7wwvuyIbWkD/WqHPsuuWq\nFqXkzHVOS/OoPNuf3auCBcMWxigrJ/GxvDFu9/iobLrPteudJGn+HZSknnCL\nTa/r\r\n=WnGO\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"c19e71624e14ef044b2e9444b9239ef707eba995","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.1_1624145186715_0.02786564702180816","host":"s3://npm-registry-packages"}},"0.38.2-danger.f25aac1ff.7":{"name":"@blitzjs/file-pipeline","version":"0.38.2-danger.f25aac1ff.7","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.2-danger.f25aac1ff.7","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"d5ebb9ffd593d6b51462ef056d8984bc47c37f69","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.2-danger.f25aac1ff.7.tgz","fileCount":26,"integrity":"sha512-xmG0cHg759R9/dB6MWflORnlx3xca4ykM+oZCFC1QLPx5lrC32E6voplMrnur6NoyDOuO46Cz4owVnV76T/LYQ==","signatures":[{"sig":"MEQCIHwVIQRRy/V5ASw+wnNnDt5HS5rip9yryUwbSM2P+KzOAiAqUL0OM2OLY1e584w6EyZ0p0TXMirhOqXPCl2DyQtWnA==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100347,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg0RmCCRA9TVsSAnZWagAAGUwP/2gfJXlJ9NoeolVk5yjU\n/4h7c4AfM6Mr9yREeWG0ROhVslX/CSataCLR2KScqXYDcbNU7y6l/nDmYDYJ\nm+hFLDIikH4a5Gf1enIr46Z1ZLaAF/ClDrsicln1y3xie4lMmmVEVHTwsKWA\nQOxRs6QrBgS0Gdl9usa4TPzVUnRn9D5ZZ4OG4YnfVVimkg3+nHaEa3sMM5t5\nfp3wfOs5Khn3Aoe1NrlxfrRvkwn+E3hI10m6ive/umyWAV6VpERgXzowREf7\niMsmORW6awnaKBU0emglXW6HZlk1HTF6C3tWo/ZqSg74gTha6Oog9lxQdxFj\nxrPCMNcRAATR0ffsoapTYmPLAqxClPawamzuwjla0CV8HfFMP3B6k2XNDBgr\nVmi1QQPuHa/t7tp3xa413bgdCUJ/C5m66Bk9yE4OYM9RNpq2cuAxukUtRvNp\nBYQd3QgpTBISt8kFAJIJPL61cUoX+NHXTVwcypgddHdSRsfjtmuqMWUTc+Qy\nIXBNHOLK5U+SvTkFF/bobeUICS2kin79jDpS/U3GYPCezeyE1SgYLNDEWTTn\nCuqwyc4c7HGgLWl4yFjLX6lwI8GS2uWjKFNoJwW91221hep99iDCYPYZ7ZfK\nSq0KNkl8J/cAS2LXWXWvnY500iTOFKoV2uPW5G0NnrmMwfa8QNE4o0E56zQX\n7LgQ\r\n=VKRC\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"f25aac1ffe408b26f2b4683edee1618a3b0dca1e","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.2-danger.f25aac1ff.7+f25aac1ff","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.2-danger.f25aac1ff.7_1624316289735_0.6331170693530728","host":"s3://npm-registry-packages"}},"0.38.2":{"name":"@blitzjs/file-pipeline","version":"0.38.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.2","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"9bbf9fc845f033b9aeabb6fa8d456086d7f80577","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.2.tgz","fileCount":26,"integrity":"sha512-K7tCJg0BajZCCIr8iSe6C6Z4nFK+5HgDpmbDVAvx1nWX9ldjmQ3wrTC9FzEy2OZffLt6bw0X8xUwORjYtEj8Cg==","signatures":[{"sig":"MEUCIE8aez7K2PyAV+O1Yk1K4B7C22ncToKvuP3LkjfkdHn7AiEAr7nHcEPGZkrKuPeAnINgi7Yp1dz7eNYOLR9qiWjv2JI=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":100289,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg1Kv4CRA9TVsSAnZWagAAwZMQAIIXk+SMd7oAjX6Y9pet\nJae0YiWg8CFLc81fsPuvcBE0U7CJbRS65zJxGQv8tG/fPvgEVkNMX+80uy+L\nj+5OR7V9WzZNi/UsgxbRIb9IiDSGDp9e7D8YMBn+VLCMNckAOtys9l/v6Nsu\nlpDeRVJRjcHeQijr57UzisRRfD82B+moZ2Sdgx21zF9nuqmnnhlWDybB/Ilg\ns339qi2xjE/NOdSuPEjHpwgGiyojgAA9csa8tS8DPwEiF8FTTMQ5h1ROvgfY\nOMhqvoPvX3FUJkFSHN5c45uNRpC9DitkQYqgUCphvHcUA6c+ejI6kR2GEqXW\n0kE0sIOW4TDfFh8h8apzUY+6Bk1GbpkV1Kongm2yUMa9TuLBHbZI+XDQzEUb\no8Mb6lMocILvhV5co/21T7de1rX2vl/IeYaYzCr+/IAplkkbJ+rnOTJZm2x+\nRD0DGyP9W0Mws+KehA0TS9DGKhVFEVG4OAF7Jotv6w42MJpgmofUsT0U3nnm\n3uoJgshKFBqkVVtxro/ExSZ1K5rYPP/pnq4VsSYKaBuEaawY7Jq9dZWEUePH\n3jYZoEUq0syTWiD0uJV730TV+fmgcqlYB4uPylXNBnmp8CF9Ey1iFFMg1lVy\nVBebtv0nMu0C4V0N+5QKOD/p6OR+f6+9LLSWdz7Y8akwU/MBYCeEZPq5QMEM\nj/5V\r\n=QFpa\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"fe8179b59587fe39db22fed946ed73a31aadae8c","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.2","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.2_1624550391781_0.7461713197753923","host":"s3://npm-registry-packages"}},"0.38.3-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.38.3-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.3-canary.0","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"54c08dfc7a22420a5abb468e4b4835197f77c44d","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.3-canary.0.tgz","fileCount":26,"integrity":"sha512-fr8oDjG0AHMBqk+tG2YKf45bX8Tf/AToPGXScilnsRQtutX2eTfOGIrNnj1ob50TbUYF6UBjHgnUlfQb6w10+g==","signatures":[{"sig":"MEUCICXBykvanh/OY0mIj8GmxcgEYkFfol8JevGeZ25yXxeTAiEAubVYvu6pA81yS5PKv8SRt4SoF8tI88/dHBRWT/r343I=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102167,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg1MN3CRA9TVsSAnZWagAAdM4QAItRZDhl/d3qNz8P3bdy\nYwvgY3P/m6FypZYcIvn/1jRE+Ea4PHsZB6wyh/T9pKWSXrCekC0/S9qVTa3J\nfUpgHzP2G20CyqEZzNWRh/gOOQccO5W5urEKoXg2bXvBNHU9FPXRlqVpYwX1\nCOWniq3AEoisR2f3hd+u97Jage/02+Xp2vo0ivIhQGr1+EaOvUV3wyAjJ+MZ\n6pBMBUeK78kBGElAcsoK2jfAxLH+NfXVph5J9u9QdAHHrkdC3Vtg2FRa7lS2\nWQPr5R3BrfW083WYXz5sEsyq3Eo/kgAjsQ1UOj2g/Gr2y6nG3lGqh+xjZgW+\nIfxsyd8/MH/j5efCDHhqeHA48K0XBM/O6QZKDaXmCMf5WzHkKCjYUzd4yYqS\nRvu/f2/Du+kjelNyztoNWe8bcWSWkcoZol1VcmTf3VmcTtupE51368jOB//O\nfppEzuQZoFr4Hk+LGJpQKTPtluVXj9gSX1lfW4fCxwXv2ex6ro6oLjoQIoyh\nhfXps5QfjBa5K8P7eW0SpVPk6g3GLj3T48YuEx7yQ4mj62BlZ4k5ohw+qqQG\nJYtB74skq8gIJQj8xMUfgr06/ZU1tJQqvZo7rADWQVWUt9qPC6jxiqWTbVqs\nAvqmRfOymX4QXgeNpDOstgufd38wulTbDwPr83jgQST+nAGHPM72Sj0gCGOW\najg0\r\n=UWLe\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"8df18f24ad7244b5bff1c5d5a3fe3f8e75a370f0","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.3-canary.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.3-canary.0_1624556407014_0.9648941947675072","host":"s3://npm-registry-packages"}},"0.38.3-danger.2a111094b.12":{"name":"@blitzjs/file-pipeline","version":"0.38.3-danger.2a111094b.12","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.3-danger.2a111094b.12","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"663735e36a7663f76d4cf3b34d0871a3ff39ba63","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.3-danger.2a111094b.12.tgz","fileCount":26,"integrity":"sha512-WakMaPjGqmZW/HPHNHzDBHd5RhmV8hTr8w1YzTvRp+4o9U/kPmrtcTxyZbCPCzfI2zQVdpMduKV2PB2hbRVYhw==","signatures":[{"sig":"MEYCIQCzSnAL8WQFZx1LJEnnWqs42GuQeehzNKDb+3Rk7/J3gAIhAIVgCrxNbbUkkiAN/gsTp1rpce5hiKJUB8oTS8OOv449","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102209,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg1T1RCRA9TVsSAnZWagAAfwoP+wUmR+JlyjAqef4ZrDvZ\nfMWkcMksCOdVHD5u9eU3d5M1XEQSkMrB5mjSGeSpiNrrIC/+mehthWPjSVwa\nY7/viP/lHksM6klv+asZAWW7HJ+lZSgKgC4PX+jd47dg+Bt4EI3G0nPsohpW\nrzc9WMgWoP83SMve3P51XDIyMAyxyym8CIlfT2dT2A4RtCoALRl2rCsSRzrt\nWcCb8yjLYO3u4SNgwDalmORjtbpXXgvtv6McVLuYyJYF/ZyLKbah+/4pj5Le\nkJggvMKjhy6t5boixztDsPiLJTLV/aZz0OWDjeLniEdn5455DGpcWxheQQ2M\n/D5DkfcsyjnTFH26KO59rbrF0HdoP9Ujs3J15k2aCoKQ/fR0Pzn4b8fH6/z2\nu9UPVaEJbp00mLcPVNsqyrb5zqX6fVvikCnyzRYyWNUn69k05GofZ8nFTcwB\nWyuGnV1Smv7bQr3tTBF49onieuCFl3i+IreIUFW2Z+dYZzZnI3f2AO0JUT3z\nd9JLEfZsxtCDhbSzFJIA03/l32uNQlDEyT/p5MhIqnuMkuwGrlSq4YVnv3ea\nAOpmbgsMz2JS1DF7tDGZ/PnR2Ze61O52Yh8oTDYv5vbZ4v5ETrquUfKXNsVA\nHb8brIYeIeuAhVx8nFDNp0nNJTDDPNXr4iAcSAHO7QLZkriiRwdUqI4CayEx\ntXXE\r\n=1iQT\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"2a111094b07df45127827f2737bfbbc0e247ea66","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.3-danger.2a111094b.12+2a111094b","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.3-danger.2a111094b.12_1624587601354_0.3093190578544207","host":"s3://npm-registry-packages"}},"0.38.3-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.38.3-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.3-canary.1","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"6ed0fabb10fb99f12a01a4a3f99268695301cdbc","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.3-canary.1.tgz","fileCount":26,"integrity":"sha512-HCXowOHvjPADANQ2jtqMOk34JH1MsQNZbmmpjsPNeMaHkNpPlS0bCOr7dJ846X3rKHAdFxbCF4MbuSEF33aKqg==","signatures":[{"sig":"MEUCIGab8NsYCvFSn/Wg0JRjmz2KNLGvpV9fqm6oUqfSoScSAiEAlogjkqF0sTTSuGy3pljnUphIUFHOPx4VB4gWcmv5vj8=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102167,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg1UkKCRA9TVsSAnZWagAA4u8P/itVXY5f5skFb2CigjGM\nAJw+jKOp+QraGMqtmd0iCugUeJCK6504hpJf/JTcjom1AhF0Swg5AIDJxOns\nOZ9NRPUhLVsyQ2N45uVYq1kLKx21hWd6/eXvvpkqdsZhWnJh+gniqaxYDmTQ\nyjhE3UftkTkniV+UQ2VU3IFNaEe9R/5J9PgZNho1bgoBtMFC7/grRag4Zqly\npoTa0NDCHJBXOo72ldva7GKnPOreRZSa43sf3zEwnnG4FC+A7a6DnObuBrsM\nEDDNqAr6d8fio9msfOcu1s5dimPW52tAghQQnjALDjbcEX3uamRZiVJuFYYO\njsuiF63jekIiNG9QyNRW7EGAJCY3bUYMQEKvSdTqgIYk/pU/+iJUIjEaUhSh\n+QFU1mnSOmDHxGR8OXSM9zkB6G/vop/R4xiHnNusJb2u5JTQKM0XCQs4XdOG\n2XvbBBkudeIW7i50uLZ6kus1V9/uyc1LXWYiBwI1vvEGrvL++bb/J1IuJOLP\nNlXgaFHK8cAYlRRbXHzfPlEoaxdQ1+T+TBrCQPvR9GhkJLCA7/mxpB1FpSG9\nG7hz3FV35bQvr9JStlhPT0e+OA/knRsMOVQL/teb09yOdS7j7D8GtphAZhZF\nfy8fqu1Pt3mhfi5Q2Dmcbl342lVWyvWmn5o/0aiDOeghAynEtcNbNIS9N5kw\ngzSe\r\n=/sUw\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"7c025e9bd7dbacd1d6fb453de24738f161539eaa","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.3-canary.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.3-canary.1_1624590601460_0.29126621528990526","host":"s3://npm-registry-packages"}},"0.38.3-danger.0df6aef3a.5":{"name":"@blitzjs/file-pipeline","version":"0.38.3-danger.0df6aef3a.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.3-danger.0df6aef3a.5","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"bbd2356632e915d295ae4a15424e7ded2e0aca18","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.3-danger.0df6aef3a.5.tgz","fileCount":26,"integrity":"sha512-dl3vpQuwMGVF2sCONUWHaxfY5hYmclrQbG5fI+YyDI9KVDzigJmR0y9tnD7p5jFJtnzg41lcT2MIJwrSwRGRcA==","signatures":[{"sig":"MEYCIQCIGd9ToO0KfEl7xmABr6HqCi3aDCGgUa6zNNZ2fyJObgIhAJ3pur4f4T582rz2QEwtn34+lABzCO+hDDTVi8wir/Pw","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102207,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg4zjvCRA9TVsSAnZWagAA7k4P/2BzrEfZGigHls55Sumu\ncns++MGzFVgnU7GsioTLXFAkCEJuDHoPKqnUHY8swoKp/vNSmKQVH4YHZmi+\nK/rdLXZYiudC/QcF40lZg0kSzPniKg5eETyBWSYwvMP0GdP2EdA3Q4WA7B+O\nc2TF4CHc6yxs8OPVuTVnrKhF0zHrnJcT2/kfQJ3XnkmZlCt5yw0cRrLeZHAe\nN4TkFtr3x0abnpMVgY+zay0EOPGGv4U4b2uFnAulrBg8XK4/OMpzs3V/9XZ/\n9zX/gJjxdS9uBiUI8Oo0oVC63OA+pzJN1gj9cJ341u1SVX6yNl1rWHd76d52\nPFTIba2lo0UcavqPPF+a9oInZG7WxaVy5YshhW2el4dsR2dpwXJAK5fUOunx\naxpg4VRFiBF9Pv5+JhcvDAl/Yjkx0FKG/BzdqVfrpMhy/+iiCmzQVayexIBI\nD3mD4CbTd+ygLnNbFc19/1tUeUEXdoR32Gxf7aKKQ7pr3Nbb0fB6mZ2Hid0X\njrfRPNu7lZ5wXwqjdU2cZQp446MugAh2OOnkTBikUJ7LnVX6cN3DNMmV2qtk\nXCInmQ9OQHp0jQyihL6M+ku0gMN8X6lmreGrFLHCANH/zDQ8bfjG/1J4vrT3\ncaE5y7mgOoMWEF2G3P/ulVeOd0ITchQKzIbYA6PzDRjBKN+vGJmAb+Z8Uku+\nAYNq\r\n=U7Qi\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"0df6aef3a3b29445f72fdd6b85b8721514a355b2","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.3-danger.0df6aef3a.5+0df6aef3a","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.3-danger.0df6aef3a.5_1625503983444_0.21060208945474446","host":"s3://npm-registry-packages"}},"0.38.3":{"name":"@blitzjs/file-pipeline","version":"0.38.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.3","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"9ab63fd954ef60249349175efebd9814112c55a6","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.3.tgz","fileCount":26,"integrity":"sha512-ADpSxkGzDln8GhSJN3o/TuIL4vMLbalknw+qTUmXlF0LHTRsfUZ+6+p8vjLQBlj/2ymDuWl2drwan1VF/ZtWcg==","signatures":[{"sig":"MEYCIQC5mPaFlVKQ0u+19+d2FrXf+qhq5FaHjnuM/b9935N8gwIhAPq+SIZQqGs1WprP6v0x9ckDKSEpjD6tk6cGCJMZ9u8a","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102149,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg44WfCRA9TVsSAnZWagAAWzMP/jerRTw0oIYaLnxUmi1L\n77nTXJ5LruliDJr1To0DjIEp6CJqIn2GMZbodKjm67hYykSLh+pEldXl26lz\nt4CkEcC+9bN6p/UGHGRUik7omkdyrtEGiibMel0/87pYgiPk8+/8Cy6V715I\n7iHKmLsN5BBvlN8GzAf3OyjYtH6JAVVz0HVzWnPFzY0g8L+kI8tMWCumJ80z\n/oG+sJIBNmDSiqAivf24Y7pOy7bV+TlweXAcdFPxX2p6qyGTro28xktX8B8O\nUQqw/3U8/uw/mjuCzGVCXaLbycASEelYdMCejyGKX61/aWNmW5/0HfgflbXa\nojvtW/R0Xw4HjT2t2DlUTMiipjTDlIvRchYXg3urIGhbeMn7G7k2X9fCcD9t\nEEsF83sj2e8s3g98UM/3+HfQOEKE1et8DAfTnHbBv2xxQ4eJDDVlMkcSLkHA\nZtXitSMVDJL89hS+MAWcv+JTXe1D29lqYQaJTb2drxtiERfEncledx+p/ktF\nqqnum2N/DYo2i57+38igqqT96/r3lRX+AZQcFndcb42Lb6YZ6PQ04JvcuzNQ\nglYiAft7ce7O8pOW1yxxcgTaAutb2j4IsTOy7JeH8ezSBRPEYkkS1OPFSZgF\ndtOb1Phth7m9HEQSbGh1x/PW6LoA6nmuiMjYG39eQB8toW2YObSvNxHIfZQW\niL/g\r\n=+5Xj\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"012fce30bb2fce0a4bfbe2728796d13f0892e9e7","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.3","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.3_1625523615453_0.0200986714691882","host":"s3://npm-registry-packages"}},"0.38.4":{"name":"@blitzjs/file-pipeline","version":"0.38.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.4","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"a40447874bea40bb45eb8d137a522017642a98a7","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.4.tgz","fileCount":26,"integrity":"sha512-G5h0crgtGaND9suoSW9Ay8Fxi/rONB4ZsCcFy6O7x3PvbasA+yosrqrDESmFLwRHdVlWtrwGw+eMzNtE/Rt6Xg==","signatures":[{"sig":"MEQCICluziICYEIu/CFGZ5PzIAi3KRYi/2K/m+gcQF0QmJCPAiA2Bwr6SVUeXetexO50/7GER6A2ck+2FWbAyYYxX7SXgw==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102149,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg5juxCRA9TVsSAnZWagAAzucP/jP5JfUdnqfus6JmUrna\nfWhuKWZXfXBG2kNykpq5rkrfS52LVWubnJSu1MGizVq7wxkBAgQsbXIDKgAO\nnOtOmfx0tWr4eJ8DgoUQ5h7Z9w1vJ6WVfhwaYNJ2SGLLMf55jq932FSaNR8I\nj+LxkAOGtfxCBZrqkqFexI98gxb5fHtacMerVhaafInF43OfjnOhPbNQD2jS\nlKFTNJ7TEIonmCgZO7hIE6tVw42suk1gKivqsRA0oaJ8KLVozVaqfzXy/vfh\nw3JSZ6aqjJv8NCok2EWzV+mY0iYDrB1V1qKoDc5hPiOK8QdFcCJB3+LGqkqI\ns6aSt26ykRNH6IDpRtGYZWOXWF+DPuZUe8RtqN+Uxxzve5bxWeNQLwlmXOS/\njTML7nXBlu+RKaHtuEWQ6NPFEx0qfgCh6Pqey36rO1/sM7F+DGXooBS+8rRx\nD+3UiIy4Fk1n/JS59Dm9C4hoWloQZBGGZIc3pFUp0HmJhucensufKTPKE81M\nZaHmC01miGlAbyKpTT0Oy3VcKI4sJo8v0kIZTRZ16YCsUW93U9wlFUWbevH1\n8lFQnmmWQTPxDDDgWUQHdfJZZ+sn+u1VkR2zBsbm249MkWbdfjsZNu5dPjPw\nroh/BhOOCvoMSjEZy218kpGNZ5qwe3Vn922orxvw4+Uy21T6ot/YnD9L6x8H\nBzy8\r\n=IrKv\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"5f2b5ac7fd5f345b792b53a12407db7636f021b2","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.4","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.4_1625701296662_0.8469363136027463","host":"s3://npm-registry-packages"}},"0.38.5":{"name":"@blitzjs/file-pipeline","version":"0.38.5","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.5","maintainers":[{"name":"skn0tt","email":"simoknott@gmail.com"},{"name":"flybayer","email":"b@bayer.ws"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"2b86c99a437c87122b80eff8454a4a84bc8c5920","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.5.tgz","fileCount":26,"integrity":"sha512-78k6VdsxMQE5Iwstb3Ym5lQMne2bl/hVwGyMc2tAWiBzXxA1q+cyConqpajjkygEkIm37T/At7/WdRpGEECUeg==","signatures":[{"sig":"MEUCIQCSfG40ba/3e/wgWpZHE5hGeQGj2qDi81Cz9Thx6VPIHwIgVgNEpYR41MS8GUwSz419hXwMVRYV5trjNrzvvk36ETU=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102149,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg53KJCRA9TVsSAnZWagAAktcP/iz7kwAv43TkEbUxIant\nnAiz5KeM0Bt3OZ5VbDSFtgc6L3R/Dyhwtx6dvPljI4pGi4XWqjwBij5+ozrC\nhSy4lCk1F55csq3HucorJQyQbsc1PxaYer+NlPr7HaBKs2PBwts+/hpRonEC\nlurTOKsHNBA39rOJfa2RHNofSaO3IGRtjnnkvrPE59QD7Q9V0CMzdxGiJjzp\ngblYpkBdqbnbDZcN1Vzojzl7Wpmx3NcfPvKtYRAVZF5PFwS+oFGOibn9p52o\n0NWezsg+tDerrt5UHcM36AcuHADce+Ubh8REqnljzUQjOVsOEx3WaobA0YBP\nMnZcq9wAZho2FbHh+n8/Z8wVZmyyT5fPrPYEjfS+7fbue7l4MQJsuoNsWER3\n8/tl2scagpnHoC8lvpeA7bc1e9tIGkQIR52Ne7VSJ4LMM5Ej5wpr46FokMbb\n1fQ0j7Xqo8UbJvEBX2zPRHBa6EAWUfaUrjaQHEhgOo0uKWHxLSsvWpYL7YPD\n9RyO1oyOlBQ4cCieSsON8xOvlL069eH4L5LI4Ih4EqhFzBaWLxc0O+x9YMXk\nxBej66yjsiBifPXLDIKhyCA0wR/5QDM43q/LUbSuZZnQt5GYBzBB1waUtVOr\noCz43zj8knF/IskyOcOKY59o2mMGcawXJJXZMzogbRzmlE50+xWjehIkPIAc\n/F/D\r\n=zF3I\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"3f901abbf01b2e4b62f0e2be564e7773b9e3c3bd","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.5","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.5_1625780873026_0.5178887787008826","host":"s3://npm-registry-packages"}},"0.38.6":{"name":"@blitzjs/file-pipeline","version":"0.38.6","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.38.6","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"skn0tt","email":"simoknott@gmail.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"46a4509dd35841fb636f9deb3684fc3225200c72","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.38.6.tgz","fileCount":26,"integrity":"sha512-VPUIHR5E1bk8AnsWvMam3LbHqWoylanBkE2yMJaZ7Hx6DZw+rEYo6ECRfQ9iJ7Yq+psgYbbyTeoASajnWS5V+A==","signatures":[{"sig":"MEYCIQCBTskG0iTut2p0gWWPTdBblQUIzh3eD06jjUMOdHsMvwIhAPsLrKbDxBQruUf6Qs/bAGYOpxSi42dW6p+iwGh8Fl84","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102149,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJg+KGPCRA9TVsSAnZWagAAlGIP/1SKfSFKkFeNOkwKRRoQ\nriF95InP//aajqV/BoTXcjGwwIOPB3WZB6fM8FSXgwz1U7cfr5YzB3iE/UjX\ncJeqVKay020GI/5SvMaVscSS6Jc8q34ut0rx8FAt5FDPokmwpUSZuAKUrUW2\nFNOuSDHWsuzNHuR2AIcVAsF78oKPqoO5vsd5NTYhQxLv81/G2ieCLpUmCU6j\nL3JjkljGb1GUQ868k4V7wDxbrJwwf5MaoMX1KakActCYUZ7kXi4Fh2Kuyfhz\nKYOQ8+82q0E70no71HBq6GjQHgRkLPW4pGTjFJ775Y6s/WGGpTiiiwZUWKK1\ngQsnXiAlnnmO3ySp3YLfwwmhHu1h1gRnzC9sZxmFdZe0kR7MFu2W3tU25VRv\nbbQzHfFYFLRkuKqTMxlAOqAn437eG7mf31IDJNQC1sazWoHjmkatZgxgvGLW\nbw9hS8+3n8ElppYZLz+cXhDhp5YJwynvYOp8o7ddch2BvHaHBP2FkTSzMQ6/\n997OVYNAwFz8QwikbeUrrzbllKkEeXnS9ivrq3enBKgTI1WjKlutxvMrMzpP\nMIjFj0ameL+i1VXkgMY25uZ651inNtcSVlGsdCgvxJijR9lNMQkTNVYL0kGX\n9uV1ebGh4FV9WbLmB6Jsts+8NQIbH8Qy8p6jLlfS/5sDZmDjOvnnNeAtKAWA\nDQsa\r\n=j2DX\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"82316682862d7eeca2924b099ffc204a5dc00f29","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.38.6","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.38.6_1626907023404_0.1360010987648217","host":"s3://npm-registry-packages"}},"0.39.0-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.39.0-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.39.0-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"skn0tt","email":"simoknott@gmail.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"5ae08b4836c0ca05222830e15edba8d925956774","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.39.0-canary.0.tgz","fileCount":26,"integrity":"sha512-5vmtruVELI+8K5cginSB3klscin2htvogTPmuwczJpJJqsPY7WtRPSfY5djYBShqm2/OXG/4iOXJl+m77/hMWA==","signatures":[{"sig":"MEQCIDTbLSDAZ9MjxkEJJ6VzPNKTv4W0iPgifd6X7+Wh1jpoAiA078aPd9xJUvgW9wePySrnOgs9q2YWRcq6memhwat9nw==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102167,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJhBVGFCRA9TVsSAnZWagAA6WIP/Rzam0zhJmNJuR56SBba\nAQgYUr8Lscb3gbwnPxqB1WSMj7O6vvh0a36Y3YVDrvWIC07rj90GXL3P2jxr\n7Aqokmd53ltZnKC3MnVMCrsO8zB6nu8Wnm89SkKNVwsiu/DzjN4CcmVXfPh7\n9fBonHGHWJ9b/RrjcchMl5l6B+y1p53u8D7hsQ/vrfUZQeOFV3UE9PbCVbFD\ngBXfFLmXxOnTK5cKwf2xJJsLmukZeKeUwqxrU+VuLKxPkb2gvRMrhZUl3ktA\nxG4YbSN8ZgtRIkbYGJisLfzzVgkpQ+a8mcpFEDuc0VeW4gGOZRXZdEwt2oAm\nawwpLOeJZWT2+Lq6KgadJyIiyJKDAnevt5pz6dvxuqccMWCHiqwHusw+XkB6\nAH2tDJV8pMVUsng7wU7we7jmR5ou51mSz52DUUBxmbOZSTzXt5Ff5xhYV7WT\nOZE9O6RVRE3i6SGorAnUVMQBLxKdgYMKLnIJKVjIngphRFYp0AH/PXmzumYb\nuk3dgE1u78dqP2u0Xvx+ZuKW6obmq8tCaAMKih8vgpwSIbhqWhuG6LFD2nIW\nm3OsFClnTWlERhzqdaU7dgul308j7DHFvcy1BjRVzjU4WDbWMBIIWjDyKcO4\nRjUz0WEebd/WNM8SP7L94iiJK1ZVA9rKWFsc4q0sK9Z+DxOmVQbtW53yRMvF\nkk5z\r\n=a4q5\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"14171543238bb353421765b2ba98db034c128746","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.39.0-canary.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.39.0-canary.0_1627738501328_0.4144484956591936","host":"s3://npm-registry-packages"}},"0.39.0":{"name":"@blitzjs/file-pipeline","version":"0.39.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.39.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"skn0tt","email":"simoknott@gmail.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"b8944ba5d77f460d23ae1982a6e3ce0408e8ff64","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.39.0.tgz","fileCount":26,"integrity":"sha512-+zs1zunIH/LvkoHpQrGn3f4SgTvN0SObKFKPmBY8c90cA1nqmTuPtl4R8KHECevwNFcg92NeFDlZzGgQZ2tcnw==","signatures":[{"sig":"MEUCIGOMsEPfMatGsoUtA9VfN4KJ+LzRdzmHdjt01vJ7ZR8lAiEArSZbBvO8uwWbTZqwQDaljKXW5128B0NytTDaUf0l5c0=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102149,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJhFqxeCRA9TVsSAnZWagAATAwP/0rNdu3WkZ/QTWV2QZUj\nAhfQ+UBD+s+d06RALHBhMK23Ve6PUPj/djbou3yLJDMOp0K/ClK1mCB2WcKI\nIJTlsIyRRe2q0Ls0LWAfrjKhY2K/OHzLJFkS4lm5Ofpfn5KlkaW7j+AgqfUH\naQBo5N5xqhTCTek6D86aL+aFF0fCWMMHA4WaKalgsccnhRfgo+QSBHOZ6bi0\nxks2oSon3X/hRRRxeHTIR/ewnnJT8sTS9lKvVzI12AHoguFB4NvbiLZ1Yw5P\nGVduPU3CFLeGEVAX0hF7GDw1bbp9L+ObKeasCIhenNaWO9bZUftLA62bcsKz\nta3ye/szS1TNf4np+V7ofy4wM18TcD/HZ4QMrqYX3bW0gAGEUKL2/n2XKg5o\n707PWwZz+UHJNuxC+6/5S+o9xY2QFe/tLjmnwcTi6KRaWlLnhrymmrXDPCWq\nFaKK9bXoXWw2GdjOpghWlC45nPPJ05LOxvRAIwiTLqHVh/ouMRCsQJj/rSu2\npnDhHw410+8+hUa4iZvUZdLZLQp42Oanpe510nxvOMHcAEw2pUwaDtcs1dIZ\nG2VO9T/UoqzQhTmxGuKn4T771VkVvqr2acYZe9RM602ao/jiUWdb/2e21M39\nXyofcjITARSF4u4BFSK2Gf1KSMPrGbYwTMqnYDZwtIl1ARwkwh7lCK5mzVwP\nCfOR\r\n=RXif\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","gitHead":"1956acf36cb1bf1be56ca85438504ff9d3deb71e","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.39.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.39.0_1628875870680_0.18897230127317988","host":"s3://npm-registry-packages"}},"0.40.0-canary.0":{"name":"@blitzjs/file-pipeline","version":"0.40.0-canary.0","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.40.0-canary.0","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"skn0tt","email":"simoknott@gmail.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"c719267e92ddb1c0e4bbc70be092e212d7c26cb4","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.40.0-canary.0.tgz","fileCount":26,"integrity":"sha512-ZdsPJhIfadPMNCI45VjRfsOENiSfQpY0TA9hrzEOMMt4Jq23ZmwIroLCecUED1No5w3oOo+L4/5Wjpc5o94j/w==","signatures":[{"sig":"MEYCIQCF+DAEmVpKmRTUW/RZ+qbJTucTLrasLkgC/2Niaw4l3QIhALhTSKaJsJGjRdvkx+ZzkvZinLJXE6tafZSfuGvMk3cM","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102741,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJhGu2uCRA9TVsSAnZWagAAlwMQAIFqT5cyZrhoyju96jLL\nwXCiaKpBrRHNR34PWS4l0LB8p3HvyOYIgbETJB9lGvu2MtIj8rn7sRPneVPA\nHBfMynr0yV3L8T6dsE3U+lXw7rwADN/PMKdPqkDQ8OarnzpWeaUq5M8j2ces\nGs6E/a5CVhOKPgULEsjOzUFDZMENM4aEaOhYVkfiIyzzu1lzldmhjVGpLWtv\nuDdLGcRFSj1ZJJxhKGM6/nPVzVQhqm8aVGN/UhfBgb5J5gHs43N/EqsfWSC0\n310lwHITCfMrrQb4RqB9dIkZPnM/jEfefHiOMbZcDawVZuV2vEIHboCGBku0\nnG8l3FEofuHhQDIbzu+U4aoLlgKEZJBXjZGlZ0akS7p6IdKpADLtHwgrPTQA\nF2BhHViMSknT5cFgN7mXqqfmwCtxZbawxCIJlEQ78k0tqIayhJ/9XV4DQfuV\nyJRDFLlbWlcDcaCGJGJdHeEV+/YR7e90oDEc2PFMI+05BxtMI5j3JAHGMC63\nLaBr9z1ER9e4+31F2/J5Yqa2N+JNd1fxlN4AbA7+Rdva4uIstXM4/JIWvPNJ\nD4BKhBeR+bixmZp4T+LugPuGkeIEGycC06zf04v5TyVDapKW0dB4+B7OOa4S\n9rDR7K32vqtK+lvAKpLce+VwM8BGazh9KC17sYHLHL3ueS0BzW4hO7tqsFk9\ncWFf\r\n=w5bU\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"44197a760c0586d496c349cb9a0b067a24940212","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.40.0-canary.0","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.40.0-canary.0_1629154734374_0.8516050073473502","host":"s3://npm-registry-packages"}},"0.40.0-danger.3e267be38.2":{"name":"@blitzjs/file-pipeline","version":"0.40.0-danger.3e267be38.2","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.40.0-danger.3e267be38.2","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"skn0tt","email":"simoknott@gmail.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"df3687f9ebe8980b299d5bdbc4fdb4becb025fe4","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.40.0-danger.3e267be38.2.tgz","fileCount":26,"integrity":"sha512-Jgoy5zgC8udDbJW4yHDI8TwmxHXdyc/eQYbU+WsQQOhpdAcoENbXUU/70ySMbp3mO5uVxnXxurjYq+kaNk5IFQ==","signatures":[{"sig":"MEQCIE8LiBWbk/+trd9/4sPyIuXNOokiq0Rd8YEsTA19AsmHAiAXs+OtLbYPO6VsjgzLZ87R8wc66Uc7jyBmCmBeYWSTFg==","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102781,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJhHEb3CRA9TVsSAnZWagAASLMP/j2g4n6FtBSrCc0kuY2o\npo+qbdBX0JPAQ7G67Ff/JmCj7ijRMprgISYxofVNusIzpN0Ol9sUyD5Xo7eW\n58fGlF789CkAiJfrkx24xOPSbga/+xXRbfSH4pheZvjag/bV1dRzpzTRmgrt\nfEOF73Zo8AMeXlAFFCZmBj7I3u9fBI0TLND6KPfi4JKUuJnsSblwD5FgtAU4\nfCLVItk0RDW8ti9SL3czDtFcLL5lAPEhpJ153c9eCUhZNtA5ptBZipHBP+4T\n52PzkvCpITfq2jVO+1ihmKHpF4cn6KxaMBKmYg6Zf2udGQ9EMgAPx8DNBEj9\nQrPPl1UdKbVgBjKhEFZ6q663qNZ62+oS/faIjT524r4Djk2FQhfsY01W62p4\nK7J5BTdB+FlUjb0ql/dfoBIUwNnA/QwYN7X/mbhBhDLnOjmzk4A58UGLI0lH\nKl78NgqiT+KkrkfL2+0IroMD8MqYgXE8+P2bQov0Y/3vgpwnMtSmAq795gIH\n5FfoShiBPzfDgXmaC1QlGxNU1pJW27Zp8i5pHbTBA+6oQLx21WoBl2+8S4Gt\nFOHayJEQ2ri1BFfEPLANh660gWEdOTzNlEA/SYUafpJzkhezrfPxwplcqFOI\nohPGU9LcS9Ox6JWYW76rdTr/jiylGcw6n2XzK+h5s+d37082proAUwK3Nchd\nRcEG\r\n=glAK\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"3e267be382245058d1c01a863a5d86f54789d874","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.40.0-danger.3e267be38.2+3e267be38","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.40.0-danger.3e267be38.2_1629243127473_0.8796167264455559","host":"s3://npm-registry-packages"}},"0.40.0-canary.1":{"name":"@blitzjs/file-pipeline","version":"0.40.0-canary.1","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.40.0-canary.1","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"skn0tt","email":"simoknott@gmail.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"80d39f133dcdad68d29ec3cfec9523247f309dc5","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.40.0-canary.1.tgz","fileCount":26,"integrity":"sha512-aj2DhvzXeh1XSoiF5oshvFxBEqrUqLZlLA8mVsG312r2gc34pPKn8CYi7KtNPYm2pL11mv1NwKeroGRH4xe/Uw==","signatures":[{"sig":"MEUCIQCFHfyyim5TnS0OsOcFQkhPvtlEeTLhbSdSE2RvyHEn4QIgfUvdYl0VD0qHa0+W2nI4xmXetWRyO0epqiRR6vkQoCw=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102741,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJhHEuZCRA9TVsSAnZWagAABBEP/iSP04pYS218Kbd/FHv2\noUIkwuvCZVcvBOvlyJlk9Kw2tPH/aIVGwjK5P+lKfc1kQ9fg00OQpnMeLwla\nX+u+hq4k+egxpCGw7HIFE97UyMQrFLbd/Qjm5aEoCgiAt2mzYMxn5+PyHshe\n6uwZRkPjCpDGnAJFXXC13KpndukAXLApPVmEtR6lsx8lX5N1XtDjlPG9PU4w\nRpV6Y3ftn0qUTzvJto9179D09WZc/EUMT+/S9+OkP/8AZpa2kf9mk8tS1zD5\nG9pwaTxslYshh9M6EQ2d4WMELyJQHKZX0S3yDNKWo6WYF81CcjfmRk2sfioS\n7csHOY6Tw3DqCq3fNxwJH/CGgJLj83GM3+BgVOj4MIiFXJUA64d46ddcJv6C\nEeKXeSt907ouFWBnJuitBsVaHo3zSGZicD+ZYBnTySsaAnJ1tG/g8BRyIktG\ncEC15ZJht5Zk2eh2UnghSWADCs6e44Qw5udeh/c6IYFs59fNKpxBWWh5Mhhi\nggH9+9Ed+Zai7ZDOIhqatOQ5RObVcLAxAYEVEjDnqLpi5vBmO/em9cD3UD7Z\nKp8JwplpAO/DkwG6a3hvPvLfDaywgOw2KnDe9xWKRdMmHDoQlasQ9w9A2qZe\n6Awd8sasoNsJHlEeRqv1Yd1WE5OZWLbOolnJVYyGfQM4g/eXNcQZf36U0hxZ\nFYkz\r\n=9lca\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"18a8d7cdd950fa16a0f6c02ff27724204fe9bc52","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.40.0-canary.1","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.40.0-canary.1_1629244313663_0.5066960965499194","host":"s3://npm-registry-packages"}},"0.40.0-canary.3":{"name":"@blitzjs/file-pipeline","version":"0.40.0-canary.3","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.40.0-canary.3","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"skn0tt","email":"simoknott@gmail.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"10c50a965c65b4169cb6fff38daf27d85ab4fb8a","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.40.0-canary.3.tgz","fileCount":26,"integrity":"sha512-FTwqpvjzYDeyXxvy1gl0PsbwvYr1rTpdYuqjZw/bBIdZVGPPTCWAuWintJ1ixtshmsnFHMpKWB1cMdLUOrkdxQ==","signatures":[{"sig":"MEYCIQCbFzPfl46/mYw0S4TVRuVEekiLZ+le/PIgQPYTbH9KWgIhAK34lKs20kkkMeJpNLzAuhhs3P5i37Dpuft/r+IDLxVt","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102741,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJhHmcuCRA9TVsSAnZWagAA1aQP/R14uOF69rSY7A8wihDY\nkcPrCyMvOcmVz4vZNqwRJl8esxz8NyP4ghCKkVjMqYZ/0SJUA6uQnIXIoMXh\n9c+dfdCAr3Bwv8ZTTjKHpX8ZZ54VmjMA4NUanOCfX2HJqK4ElmWDMPdJ48Gl\nnJXgcUj0igW7gDw9LyYSznYva4dJ/yf8WV0ZSsDpF8DQNxuUqjrMiKVviQjz\n+xttC7/fVeuaS0enBq6pj/l29jzG+vP/FYNfD1idzYdxz6WjY9jIDvNiahSK\nNeEjYZbrLavUfxvzyB5moEMbz60xsDnLU3j70a9MaPcmBCA0jEL3hPdiREJM\nkGOKfUxExUISO3Rxx8gW7Kq6hYJdCqs/L8zLdLQ2ikHDIY2Ijpdxhp0to5To\nifSLjKS3e7HsWkhmp/5hL130nWrYlOvRFLKcVuioGak1weKtA8B7ThesvH6m\n9tpzmOVR12AoP6vGGZSQc9MMsiOEUdatF5BGm72SQdzLup36TRjaI1VXVmeg\nVf00p4bcJDbDLbIHeG3NhzjxwvgaZo6P7NefefrI+czn/CVxaAnc78WgXNMj\nxmnAPUckXVUFZuqI0TMPVzlHCOw4C7d0m6IkPrT8rWM1L2FHLeQjr+fdcT5Q\nQ6VvXnCYBFnIaJR8vA326hYGpqDFo5RMZp5ZGsJQpDQOfNFQG4zi8/Z7eSku\ntCRJ\r\n=fHv2\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"f1c458fe129ea2d5e0de798e7088f4f14a5c42b2","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.40.0-canary.3","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.40.0-canary.3_1629382446433_0.4288386357616538","host":"s3://npm-registry-packages"}},"0.40.0-canary.4":{"name":"@blitzjs/file-pipeline","version":"0.40.0-canary.4","keywords":["blitz","display"],"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","_id":"@blitzjs/file-pipeline@0.40.0-canary.4","maintainers":[{"name":"flybayer","email":"b@bayer.ws"},{"name":"skn0tt","email":"simoknott@gmail.com"}],"homepage":"https://github.com/blitz-js/blitz#readme","bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"dist":{"shasum":"27e25655752a9ef14740415748dbfa8ae1a1aa09","tarball":"https://registry.npmjs.org/@blitzjs/file-pipeline/-/file-pipeline-0.40.0-canary.4.tgz","fileCount":26,"integrity":"sha512-El44Ft+fnr8BC/bzL0ZL5ubsnKxTBtnfnyMEcGxE2s5v8tZsnms63JyBplq/l40AYDXXj05zZmMIlarMbnnHSg==","signatures":[{"sig":"MEYCIQCsH4G3nE7GLdQ47Ct2INuQ5FUPYMar5eEA+xXJTv/61gIhALHDSHEAbwhsAQ/uGZDKdaBYS8UmdtL0KeXQSStlGzt8","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":102741,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJhH7QuCRA9TVsSAnZWagAAwScP/2GceEzr9ZfX6HqFK5Qz\ngVunl//Sx4nxbbPBTluzKPQ6G4i7yKUurtQOIZYobk8Q/BnmX5pfU/iBVl1U\n18WVf8HVdM+BSiOvp+CfQRwEq3mtQ4fFAwgjg7Rf6mroU1o8xaVedJJQEybe\nVRvaU3UykeBqOSQNhyosSyi1CtWe6fZqny6CWqPmCPlRbXmgnGLoKSHmuPvw\nOx5pd43OVpS+yAOFNrftgqls/g7p3UI7J1bRXoYWY+xTQEd/WQBimehFoyxu\nbIRhs18l/U2G1r08tKY/rT/Ia/0aHVSRd2L44vrnsRbkHLFswdKQuw5BHfbd\nVXNW3V1tPn38x7Nzm3tswv12t+9WLpugBzsm2nkBgKhaiIq/Z8LZDnXP7E2P\n1wJeeQnrIornqaj73Kh8nnYJ8jrwd3OV8bZ3s4+GrY+GyRTImysDcyYeUswK\na2C0Ne/EAXGipzUFvMDbHtfR/70o4nTnPX5tGdGrvc/Y3U/CxwSXZDWYU2rz\nlbCK7xgeUsW9vxMNibeTsIwNjU8r1ua3ePklbJFUaqlu/hUnX5a9aMA2girh\nxiuleHjc8+k4ofp7k7ctBFnUtWMe9L5oz37CAZ9q39QAPfJ6JguHV/vJU21o\nVmZWRhEcnNKs587auNBueNCzg0llza+Lb0Q0rZdqnz8fiyiQlzCl5SkT988l\nXM5U\r\n=Z86L\r\n-----END PGP SIGNATURE-----\r\n"},"main":"dist/blitzjs-file-pipeline.cjs.js","types":"dist/blitzjs-file-pipeline.cjs.d.ts","module":"dist/blitzjs-file-pipeline.esm.js","readme":"# `@blitzjs/file-pipeline`\n\n## File Transformation Pipeline\n\n### Design goals\n\n- Stages: Related logic needs to live together\n- Everything is a stream\n- Efficiency\n- Parallel processing\n- Cleaner Architecture for Dirty restart functionality\n- Agnostic input file watcher / glob\n- Simplify tests\n\nThis package provides a gulp based pipeline for transforming files from a source to a destination across a series of stages.\n\n```ts\nimport {transformFiles} from '@blitzjs/file-pipeline'\nimport through from 'through2'\nimport File from 'vinyl'\n\n// This is an example of a stage that does nothing to the\n// files as they pass through the pipeline\nconst myStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\nconst mySecondStage = () => ({\n stream: through.obj((file:File, _, next) => {\n // Normally transformation will take place here\n next(null, file)\n })\n})\n\n// Files start off at the source\nconst src = '/path/to/src'\n// Pass through the stages one by one\nconst stages = [\n myStage,\n mySecondStage\n]\n// Then end up at the destination\nconst dest = '/path/to/dest'\n\n// We can set various options they are all optional\nconst options = {\n // This indicates if the file watcher will be turned on\n watch:true,\n // this is a list of source files globs to ignore\n ignore: [],\n // this is a list of source files globs to include\n include: ['**/*']\n // pass in an optional transform stream that will be used as an event bus\n bus: someTransformStream\n}\n\n// run the transform\ntransformFiles(src, stages, dest, options),\n```\n\n## Stages\n\nStages are how you provide special behaviour to your file-pipeline.\n\nThe anatomy of your stage looks like this:\n\n```ts\nfunction myStage({\n // Stage config holds the basic info you need for the stage\n config: {\n // src folder\n src,\n // dest folder\n dest,\n // current working directory\n cwd,\n // include globs\n include,\n // ignore globs\n ignore,\n // if we are in watch mode\n watch,\n },\n // Input writable stream - use input.write(file) to send a file the input of the pipeline\n input,\n // Event bus stream - use this to send events to listeners within and outside of the pipeline\n bus,\n // Get the input cache.\n // This is an object that contains cached objects for all the files ingested.\n // Use this for things that require lists of files\n getInputCache,\n}: StageArgs) {\n // Create some kind of transform stream\n const stream = createSomeKindOfTransformStream()\n\n // Ready - is an object that will be merged with all other\n // Stages and returned in a promise by transformFiles()\n const ready = {foo: \"This will appear in the object returned by transformation promise\"}\n\n // Export the stream and the ready info\n return {stream, ready}\n}\n```\n\n# Why Streams?\n\nInitially, Blitz will be used by people with small projects however as the number files and throughput increases we will need to use an architecture that allows for large parallel throughput with low memory consumption. Node is built on streams as a primitive so it makes sense to utilize what is available. The Gulp ecosystems provide several tools for managing streams of files so that makes sense to use those tools when available. Because refactoring to streams later would be extremely difficult and painful not starting with streams would be a design mistake.\n\n# Why not RxJS?\n\nRxJS could be a good match for streaming architectures and introduces some really powerful tools for managing stream operations. As we are using object streams it would also possibly simplify some of the boilerplate using RxJS. However, certain operators in RxJS can be inapproachable for newer developers and tend to encourage too much abstraction. It is also an extra dependency that increases the learning surface of the codebase and as we are stuck with understanding basic node streams, in any case, it makes sense to avoid RxJS until absolutely necessary.\n\n## File Transform Pipeline\n\n\n\n# Stream helpers\n\nSo Node streams are a little incompatible on old versions of Node and there are a few compatibility libs we are using to help us work with streams.\n\nhttps://www.freecodecamp.org/news/rxjs-and-node-8f4e0acebc7c/\n\nHelper Libs\n\n- Pipe - [pump](https://npmjs.com/package/pump)\n- Pipeline - [pumpify](https://npmjs.com/package/pumpify)\n- Through - [through2](https://npmjs.com/package/through2)\n- Concat - [concat-stream](https://npmjs.com/package/concat-stream)\n- Parallel - [parallel-transform](https://npmjs.com/package/parallel-transform)\n- Node Compat - [readable-stream](https://npmjs.com/package/readable-stream)\n\n# A good way to work with streams\n\nA pattern we have found that works well is using a constructor function to accept connectors and return a stream as well as any shared data you need to provide to other components connectors. You will see this a lot around the synchronizer.\n\n```ts\ntype CreatorFn = ConnectionsOrConfig => StreamAsWellAsSharedData\n```\n\nAn example might look like this:\n\n```ts\n// Config -> Merged Glob && FileWatcher\nconst source = agnosticSource({cwd: src, include, ignore, watch})\n\n// you can then pipe the stream to a pipeline\npipe(source.stream, fileTransformPipeline)\n```\n\nThe reason we don't just return a stream is that often we need to return other data and share it elsewhere, for example, to analyze input file structure in the pages rule we use a file cache.\n\n```ts\n// Here\nconst fileCache = createFileCache(config)\nconst pageRule = createPageRule(fileCache.cache)\n\npipeline(\n fileCache.stream, // manages the fileCache to be used by other streams\n // ...\n pageRule.stream, // has access to the fileCache\n)\n```\n\n# View rendering and error handling\n\nThe cli view is provided by a stream which accepts Events which it manages and displays. This is responsible for rendering stuff to the view.\n\nIf you push an Error to the transform stream `next(new Error)` an Error Event will be sent over the event bus. This can get handled by some event handling logic in your Stage.\n\n# Evented Vinyl Files\n\nEvented Vinyl Files are [Vinyl Files](https://github.com/gulpjs/vinyl) with events attached to them\n\n```ts\nconst isDelete = (file) => file.isNull() && file.event === \"unlink\"\n\n// The input file at '/path/to/foo' was deleted\n// This can be transformed during the process phase\nreturn new Vinyl({\n path: \"/path/to/foo\",\n content: null,\n event: \"unlink\",\n})\n```\n\n```ts\n// Add file at '/path/to/foo'\nnew Vinyl({\n path: \"/path/to/foo\",\n content: someContentStream,\n})\n```\n\n# Input agnostic\n\nPipeline is input agnostic ie. it should not matter if it comes from watch or a folder glob so to help with that we have created an agnostic input stream that takes glob config and returns a file stream. It consumes input from both chokidar and vinyl-fs.\n\n# Optimization\n\nInput manages inputting of evented vinyl file. Files that have already been processed or are currently being processed should not be processed again. Here we try and manage a running list of files to work on based on the hash of their filename and mtime.\n\n# Analysis\n\nSome types of analysis need a list of all the files other types do not\n\nAnalysis needs to be done in stream as new information comes in. Eg. when someone renames a file that file goes to the analysis engine which works out invariants as they occur without requiring a sweep of the entire file system.\n\nFor this, we can create file caches which represent projections of the file system and update based on input file events.\n\n# Stages\n\nStage streams represent specific things we need the `file-pipeline` to do\n\nPossible things it can do:\n\n- Change its path or contents\n- Drop the file from further processing. Don't copy it.\n- Add new files to the input stream - Associating the new files with the original\n- Write an error to the error stream\n\n```ts\nimport {through} from './streams'\n\n// Typical Stage\nexport default myStage({config, input, getInputCache}) => {\n const service = createSomeService()\n\n // This is an incremental file cache that\n // gets built as Files are read\n const cache = getInputCache()\n\n // You should not write to the file system use input.write() instead to send\n // a new file down the pipelin\n if (!pathExistsSync(resolve(config.src, 'blitz.config.js'))) {\n input.write(\n new File({\n path:resolve(config.src, 'blitz.config.js'),\n content:Buffer.from('Hello World')\n })\n )\n }\n\n const stream = through.obj(function (file, enc, next) {\n // You can test for changes in the input cache\n if (cache.filter(/next\\.config\\.js$/.exec).length > -1) {\n const err = new Error('Cannot have next config!')\n err.name = 'NextConfigError'\n next(err) // Will cause an Error Event to be fired on the event bus\n }\n\n // process file in some way\n file.path = file.path.toUpperCase()\n\n // you can push to the stream output (note you cannot use arrow fns)\n this.push(file)\n\n // You can push new files to the output too but this is a little dodgy\n // One issue here is that this file will not be registered as a piece\n // of work and this may have unintended consequences.\n // This is still ok in many situations but it is safer to write to the input.\n // (note input.write vs this.push)\n this.push(\n new File({\n path: '/path/to/foo',\n content: Buffer.from('This is a file')\n })\n )\n\n // or send file onwards to be written this does the same thing as this.push()\n next(null, file)\n })\n\n // provide an object that will be merged with other ready\n // objects to form the return of the transformFiles return promise value\n const ready = {\n foo: 'I am foo'\n }\n\n return {\n stream,\n ready,\n service, // provide that service to consumers outside the stream\n }\n}\n```\n","gitHead":"c99b1fc9dbc815ed8df2d39fccf76112837f4fe5","scripts":{"test":"jest","test:watch":"jest --watch"},"_npmUser":{"name":"flybayer","email":"b@bayer.ws"},"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"_npmVersion":"lerna/4.0.0/node@v12.20.0+x64 (darwin)","description":"Display package for the Blitz CLI","directories":{},"_nodeVersion":"12.20.0","dependencies":{"pump":"3.0.0","chalk":"^4.1.0","from2":"2.3.0","vinyl":"2.2.1","lodash":"^4.0.0","rimraf":"^3.0.2","gulp-if":"3.0.0","pumpify":"2.0.1","chokidar":"3.5.1","fs-extra":"^9.1.0","through2":"4.0.2","vinyl-fs":"3.0.3","micromatch":"4.0.2","vinyl-file":"3.0.0","merge-stream":"2.0.0","@blitzjs/display":"0.40.0-canary.4","flush-write-stream":"2.0.0","parallel-transform":"1.2.0"},"_hasShrinkwrap":false,"readmeFilename":"README.md","_npmOperationalInternal":{"tmp":"tmp/file-pipeline_0.40.0-canary.4_1629467694101_0.3668858055032398","host":"s3://npm-registry-packages"}}},"time":{"created":"2020-06-05T15:24:10.340Z","modified":"2025-07-02T15:24:37.772Z","0.13.0-canary.2":"2020-06-05T15:24:10.739Z","0.13.0-canary.3":"2020-06-06T06:08:02.216Z","0.13.0":"2020-06-06T09:27:28.982Z","0.14.0-canary.0":"2020-06-13T15:38:47.960Z","0.14.0":"2020-06-13T16:01:09.633Z","0.14.1":"2020-06-15T02:36:04.289Z","0.14.2-canary.0":"2020-06-15T06:51:13.935Z","0.15.0":"2020-06-16T10:49:47.244Z","0.15.1-canary.0":"2020-06-17T04:55:05.259Z","0.15.1":"2020-06-17T11:14:04.743Z","0.15.2-canary.0":"2020-06-20T05:44:31.066Z","0.15.2-canary.1":"2020-06-20T07:19:32.479Z","0.15.2-canary.2":"2020-06-20T08:37:19.666Z","0.15.2-canary.3":"2020-06-20T08:49:29.376Z","0.15.2":"2020-06-20T09:00:23.723Z","0.15.3":"2020-06-23T13:41:56.086Z","0.16.0":"2020-07-01T09:31:11.446Z","0.16.1":"2020-07-02T17:04:29.090Z","0.16.2":"2020-07-06T14:42:11.300Z","0.16.3":"2020-07-08T10:18:02.897Z","0.16.4":"2020-07-13T12:34:08.030Z","0.16.5-canary.0":"2020-07-16T14:02:20.593Z","0.16.5-canary.1":"2020-07-18T09:30:16.461Z","0.16.5-canary.2":"2020-07-20T13:47:54.744Z","0.16.5-canary.3":"2020-07-21T03:19:04.375Z","0.16.5-canary.4":"2020-07-21T09:14:12.776Z","0.16.5-danger.4":"2020-07-22T11:15:42.761Z","0.16.5-canary.5":"2020-07-22T15:23:16.647Z","0.16.5-danger.3":"2020-07-23T14:44:54.485Z","0.16.5-danger.e654019.4":"2020-07-23T15:14:07.742Z","0.16.5-danger.027430a.5":"2020-07-23T15:19:24.854Z","0.16.5-danger.c53e404.6":"2020-07-23T15:28:29.000Z","0.16.5-canary.7":"2020-07-25T11:50:28.021Z","0.16.5-canary.8":"2020-08-02T21:31:48.673Z","0.16.5-danger.0ca895c.0":"2020-08-02T21:52:06.347Z","0.17.0":"2020-08-02T22:08:28.570Z","0.17.1-canary.0":"2020-08-04T21:25:09.563Z","0.17.1-canary.1":"2020-08-04T21:29:53.537Z","0.17.1-canary.2":"2020-08-05T01:59:02.381Z","0.17.1-canary.3":"2020-08-08T22:42:11.123Z","0.17.1-danger.518d9666.8":"2020-08-12T21:28:03.957Z","0.17.1-danger.3b5aa900.11":"2020-08-12T22:05:51.323Z","0.17.1-canary.4":"2020-08-12T23:22:54.383Z","0.17.1-canary.5":"2020-08-18T02:45:22.832Z","0.17.1-canary.6":"2020-08-19T01:09:11.149Z","0.17.1-canary.7":"2020-08-19T21:36:09.176Z","0.18.0":"2020-08-20T00:41:01.581Z","0.18.1-canary.0":"2020-08-30T19:32:36.773Z","0.18.1-canary.1":"2020-08-30T19:42:49.965Z","0.19.0":"2020-08-30T20:03:24.646Z","0.20.0":"2020-08-31T20:23:20.220Z","0.21.0":"2020-09-01T20:22:24.032Z","0.21.1":"2020-09-02T17:23:30.467Z","0.21.2-canary.0":"2020-09-09T14:05:56.561Z","0.21.2-canary.1":"2020-09-09T16:30:26.504Z","0.21.2-canary.2":"2020-09-11T15:10:40.907Z","0.22.0":"2020-09-11T21:48:42.060Z","0.22.1":"2020-09-12T18:03:19.133Z","0.22.2-canary.0":"2020-09-15T02:07:39.560Z","0.22.3-canary.0":"2020-09-20T02:02:02.489Z","0.23.0":"2020-09-22T23:28:56.622Z","0.23.1-canary.0":"2020-09-24T02:27:02.809Z","0.23.2-canary.0":"2020-09-27T02:21:23.586Z","0.23.1":"2020-09-28T21:20:06.329Z","0.23.2":"2020-09-28T21:27:02.067Z","0.23.2-canary.1":"2020-09-28T21:32:33.042Z","0.23.2-canary.2":"2020-09-28T22:58:38.568Z","0.23.2-canary.3":"2020-09-30T15:50:13.937Z","0.23.3":"2020-10-01T18:35:07.052Z","0.23.2-danger.38af6c59.3":"2020-10-02T01:04:29.620Z","0.23.2-danger.d81f84c9.4":"2020-10-02T02:04:15.533Z","0.24.0-canary.0":"2020-10-02T20:29:31.788Z","0.23.4":"2020-10-02T20:55:29.866Z","0.24.0-canary.1":"2020-10-03T01:58:06.287Z","0.24.0-canary.2":"2020-10-05T23:13:30.444Z","0.24.0-canary.3":"2020-10-06T15:23:58.243Z","0.24.0-danger.e5ead166.1":"2020-10-06T15:54:41.864Z","0.24.0-canary.4":"2020-10-06T16:11:40.339Z","0.24.0-danger.e78a8210.1":"2020-10-06T22:36:27.484Z","0.24.0":"2020-10-06T23:24:59.669Z","0.24.1":"2020-10-07T00:33:35.313Z","0.24.2":"2020-10-14T01:10:55.852Z","0.24.3":"2020-10-16T16:08:05.209Z","0.25.0-canary.0":"2020-10-21T14:11:35.163Z","0.25.0-canary.1":"2020-10-28T02:18:22.680Z","0.25.0-canary.2":"2020-11-05T02:58:20.130Z","0.25.0-canary.3":"2020-11-10T02:09:22.280Z","0.25.0-canary.4":"2020-11-10T02:29:00.502Z","0.25.0-canary.5":"2020-11-10T02:37:37.570Z","0.25.0-canary.6":"2020-11-11T01:56:37.489Z","0.25.0":"2020-11-11T23:33:56.140Z","0.25.1-canary.0":"2020-11-18T02:47:38.665Z","0.25.1-canary.1":"2020-11-19T02:35:23.178Z","0.25.1-canary.2":"2020-11-21T00:44:31.699Z","0.25.1-canary.3":"2020-11-21T02:31:39.823Z","0.26.0":"2020-11-24T16:00:55.197Z","0.26.1":"2020-11-27T17:22:38.221Z","0.26.2":"2020-11-27T20:38:31.263Z","0.27.0-canary.0":"2020-11-28T18:30:03.356Z","0.27.0-canary.1":"2020-12-01T18:01:35.058Z","0.27.0-canary.2":"2020-12-03T23:48:57.420Z","0.27.0-canary.3":"2020-12-03T23:58:11.361Z","0.27.0":"2020-12-06T00:07:49.718Z","0.27.1":"2020-12-11T00:20:15.494Z","0.28.0-canary.0":"2020-12-17T21:24:17.925Z","0.28.0-canary.1":"2020-12-23T19:22:00.628Z","0.28.0-canary.2":"2021-01-01T22:36:46.349Z","0.28.0":"2021-01-04T20:20:37.744Z","0.29.0-canary.0":"2021-01-08T23:31:01.898Z","0.29.0":"2021-01-12T22:32:31.273Z","0.29.1-danger.406c2ae7.0":"2021-01-12T23:45:16.117Z","0.29.1-danger.796e93d9.1":"2021-01-13T00:00:02.489Z","0.29.1-danger.f783246f.2":"2021-01-13T00:04:04.311Z","0.29.1":"2021-01-13T00:29:36.458Z","0.29.2-danger.cfee2e9e.1":"2021-01-13T22:19:39.449Z","0.29.2-danger.9c6283fd.2":"2021-01-13T22:37:03.550Z","0.29.2-danger.c73eebfc.2":"2021-01-13T22:42:46.696Z","0.29.2":"2021-01-13T23:32:28.479Z","0.29.3-danger.28bf8b38.5":"2021-01-15T22:20:03.244Z","0.29.3-danger.d7fef62e.6":"2021-01-15T22:24:05.143Z","0.29.3-danger.2b28990a.7":"2021-01-15T22:54:03.966Z","0.29.3-danger.5b44344b.8":"2021-01-15T23:02:36.754Z","0.29.3-danger.2c0a6847.17":"2021-01-19T16:39:52.042Z","0.29.3":"2021-01-20T22:23:21.498Z","0.29.5":"2021-01-26T16:52:51.649Z","0.29.6":"2021-01-27T15:16:02.801Z","0.29.7":"2021-01-27T16:49:52.381Z","0.30.0-canary.0":"2021-01-27T17:31:57.146Z","0.30.0-canary.1":"2021-01-27T22:14:11.615Z","0.30.0-canary.2":"2021-01-28T02:21:48.500Z","0.30.0-canary.3":"2021-01-29T18:55:26.501Z","0.30.0-canary.4":"2021-01-30T02:29:10.051Z","0.30.0-canary.5":"2021-02-05T20:52:06.940Z","0.30.0-canary.6":"2021-02-07T00:06:43.611Z","0.30.0-danger.4b23a38d.5":"2021-02-09T22:26:57.246Z","0.30.0-danger.62a62ee1.6":"2021-02-09T22:53:28.345Z","0.30.0-danger.bc7cfa45.7":"2021-02-09T22:59:09.481Z","0.30.0-danger.15c47545.8":"2021-02-09T23:06:27.565Z","0.30.0-canary.7":"2021-02-10T00:37:03.091Z","0.30.0-canary.8":"2021-02-11T00:22:54.737Z","0.30.0-canary.9":"2021-02-11T20:57:10.329Z","0.30.0-canary.10":"2021-02-11T22:17:45.216Z","0.30.0-danger.c389a7ad.0":"2021-02-11T23:50:00.622Z","0.30.0-canary.11":"2021-02-12T00:54:59.879Z","0.30.0-canary.12":"2021-02-12T19:10:51.363Z","0.30.0-canary.13":"2021-02-14T02:25:25.526Z","0.30.0-canary.14":"2021-02-15T19:33:35.675Z","0.30.0-canary.15":"2021-02-17T03:25:35.110Z","0.30.0":"2021-02-17T13:50:07.966Z","0.30.1":"2021-02-19T02:14:54.701Z","0.30.2":"2021-02-20T20:39:44.970Z","0.30.3":"2021-02-23T21:43:57.515Z","0.30.4":"2021-02-24T20:35:04.065Z","0.30.5":"2021-02-25T16:27:25.670Z","0.30.6":"2021-02-27T22:16:44.678Z","0.30.6-danger.8c211d9b.10":"2021-02-27T23:57:31.946Z","0.30.7-danger.689b7beb.20":"2021-03-02T21:42:48.595Z","0.30.7-danger.a2f18c44.21":"2021-03-02T21:55:49.480Z","0.30.7":"2021-03-03T22:35:08.776Z","0.30.7-danger.c13a2382.37":"2021-03-04T17:57:45.821Z","0.31.0":"2021-03-04T21:41:03.588Z","0.31.1-danger.7bc800c2.6":"2021-03-05T16:21:21.610Z","0.31.1":"2021-03-05T18:06:21.680Z","0.31.2-danger.d90edd13.4":"2021-03-06T20:57:50.186Z","0.31.2-danger.7a88537f.5":"2021-03-08T21:30:21.141Z","0.32.0":"2021-03-09T22:00:08.713Z","0.32.1":"2021-03-10T17:35:10.273Z","0.33.0":"2021-03-17T19:21:33.706Z","0.33.1":"2021-03-27T15:16:41.852Z","0.30.7-danger.d5caf8725.39":"2021-04-02T15:52:34.028Z","0.33.2-danger.776e20c3f.37":"2021-04-05T20:40:11.242Z","0.33.2-danger.192318af2.43":"2021-04-05T21:29:54.074Z","0.34.0-canary.0":"2021-04-05T22:07:22.358Z","0.34.0-canary.1":"2021-04-06T20:46:53.012Z","0.34.0-canary.2":"2021-04-07T19:40:18.194Z","0.34.0-danger.bda22a774.8":"2021-04-10T21:43:29.747Z","0.34.0-danger.c9dd35ad4.9":"2021-04-10T22:02:12.107Z","0.34.0-danger.625089a0b.9":"2021-04-10T22:05:41.530Z","0.34.0-canary.3":"2021-04-10T22:24:20.564Z","0.34.0-danger.5dbd9a26f.9":"2021-04-13T19:15:59.505Z","0.34.0-danger.e8dd18645.10":"2021-04-13T19:20:04.100Z","0.34.0-y.0":"2021-04-15T16:54:23.239Z","0.34.0-canary.4":"2021-04-15T17:02:42.069Z","0.34.0-danger.2a307c801.1":"2021-04-15T20:31:34.185Z","0.34.0-canary.5":"2021-04-15T20:47:42.238Z","0.34.0":"2021-04-16T19:43:21.189Z","0.34.1":"2021-04-19T23:53:14.675Z","0.34.2-danger.1079ed84d.3":"2021-04-20T15:51:12.615Z","0.34.2-canary.0":"2021-04-21T15:58:51.227Z","0.34.2":"2021-04-21T16:02:48.162Z","0.34.3":"2021-04-21T16:17:04.109Z","0.35.0-canary.0":"2021-04-21T21:46:12.103Z","0.35.0-danger.fa264b8d8.4":"2021-04-25T14:09:45.032Z","0.35.0-canary.1":"2021-04-26T19:12:28.877Z","0.35.0-canary.2":"2021-04-27T19:50:18.438Z","0.34.4-danger.f06f4b853.6":"2021-04-29T17:17:40.667Z","0.35.0-canary.3":"2021-04-29T22:39:44.654Z","0.35.0-danger.91b725cb3.3":"2021-05-03T22:01:52.958Z","0.35.0-danger.22054c4cf.3":"2021-05-03T22:27:48.346Z","0.35.0-canary.4":"2021-05-04T20:42:39.757Z","0.35.0-canary.5":"2021-05-05T19:46:56.912Z","0.35.0-canary.6":"2021-05-10T22:42:23.757Z","0.35.0":"2021-05-17T22:46:34.440Z","0.35.1":"2021-05-26T21:15:29.486Z","0.35.2":"2021-05-26T21:38:58.238Z","0.36.2":"2021-05-28T21:15:03.460Z","0.36.3-danger.3b2531081.3":"2021-05-29T22:36:24.759Z","0.36.3":"2021-05-29T22:47:54.948Z","0.36.4":"2021-06-02T17:22:02.020Z","0.36.5-danger.7bda601cd.11":"2021-06-07T21:17:38.469Z","0.36.5-danger.e4504f3fa.12":"2021-06-07T21:22:43.584Z","0.36.5-danger.73705b625.14":"2021-06-07T22:16:43.279Z","0.36.5-danger.5fb0f166b.15":"2021-06-07T22:22:23.105Z","0.36.5-danger.e7acb2375.16":"2021-06-07T22:40:33.847Z","0.37.0":"2021-06-08T18:31:56.312Z","0.37.1-danger.0c154426a.8":"2021-06-11T16:54:09.821Z","0.37.1-danger.e2b6dd277.30":"2021-06-16T20:22:06.668Z","0.37.1-danger.7c1b90c56.33":"2021-06-16T22:09:57.363Z","0.37.1-danger.a640b1112.35":"2021-06-16T23:37:21.492Z","0.38.0":"2021-06-17T13:31:09.912Z","0.38.1":"2021-06-19T23:26:26.837Z","0.38.2-danger.f25aac1ff.7":"2021-06-21T22:58:10.116Z","0.38.2":"2021-06-24T15:59:51.905Z","0.38.3-canary.0":"2021-06-24T17:40:07.201Z","0.38.3-danger.2a111094b.12":"2021-06-25T02:20:01.589Z","0.38.3-canary.1":"2021-06-25T03:10:01.739Z","0.38.3-danger.0df6aef3a.5":"2021-07-05T16:53:03.550Z","0.38.3":"2021-07-05T22:20:15.624Z","0.38.4":"2021-07-07T23:41:36.756Z","0.38.5":"2021-07-08T21:47:53.213Z","0.38.6":"2021-07-21T22:37:03.563Z","0.39.0-canary.0":"2021-07-31T13:35:01.568Z","0.39.0":"2021-08-13T17:31:10.813Z","0.40.0-canary.0":"2021-08-16T22:58:54.537Z","0.40.0-danger.3e267be38.2":"2021-08-17T23:32:07.616Z","0.40.0-canary.1":"2021-08-17T23:51:53.782Z","0.40.0-canary.3":"2021-08-19T14:14:06.593Z","0.40.0-canary.4":"2021-08-20T13:54:54.255Z"},"bugs":{"url":"https://github.com/blitz-js/blitz/issues"},"author":{"url":"https://twitter.com/flybayer","name":"Brandon Bayer","email":"b@bayer.ws"},"license":"MIT","homepage":"https://github.com/blitz-js/blitz#readme","keywords":["blitz","display"],"repository":{"url":"git+https://github.com/blitz-js/blitz.git","type":"git"},"description":"Display package for the Blitz CLI","maintainers":[{"email":"mina.youssef@gmail.com","name":"mabadir"},{"email":"b@bayer.ws","name":"flybayer"},{"email":"info@simonknott.de","name":"skn0tt"},{"email":"siddh.suresh@gmail.com","name":"siddhsuresh"}],"readme":"","readmeFilename":""}