{"_id":"@aghorbani/llama.rn","name":"@aghorbani/llama.rn","dist-tags":{"latest":"0.3.12"},"versions":{"0.3.12":{"name":"@aghorbani/llama.rn","version":"0.3.12","description":"React Native binding of llama.cpp","main":"lib/commonjs/index","module":"lib/module/index","types":"lib/typescript/index.d.ts","react-native":"src/index","source":"src/index","scripts":{"bootstrap":"./scripts/bootstrap.sh","docgen":"typedoc src/index.ts --plugin typedoc-plugin-markdown --readme none --out docs/API","test":"jest","typecheck":"tsc --noEmit","lint":"eslint \"**/*.{js,ts,tsx}\"","prepack":"bob build","release":"release-it","example":"yarn --cwd example","build:ios":"cd example/ios && xcodebuild -workspace RNLlamaExample.xcworkspace -scheme RNLlamaExample -configuration Debug -sdk iphonesimulator CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ GCC_OPTIMIZATION_LEVEL=0 GCC_PRECOMPILE_PREFIX_HEADER=YES ASSETCATALOG_COMPILER_OPTIMIZATION=time DEBUG_INFORMATION_FORMAT=dwarf COMPILER_INDEX_STORE_ENABLE=NO","build:android":"cd example/android && ./gradlew assembleDebug","clean":"del-cli example/ios/build"},"keywords":["react-native","ios","android","large language model","LLM","Local LLM","llama.cpp","llama","llama-2"],"repository":{"type":"git","url":"git+https://github.com/mybigday/llama.rn.git"},"author":{"name":"Jhen-Jie Hong","email":"developer@jhen.me","url":"https://github.com/mybigday"},"license":"MIT","bugs":{"url":"https://github.com/mybigday/llama.rn/issues"},"homepage":"https://github.com/mybigday/llama.rn#readme","publishConfig":{"registry":"https://registry.npmjs.org/"},"devDependencies":{"@commitlint/config-conventional":"^17.0.2","@evilmartians/lefthook":"^1.2.2","@fugood/eslint-config-react":"^0.5.0","@react-native-community/eslint-config":"^3.0.2","@release-it/conventional-changelog":"^5.0.0","@types/jest":"^28.1.2","@types/react":"~17.0.21","@types/react-native":"0.70.0","commitlint":"^17.0.2","del-cli":"^5.0.0","eslint":"^8.4.1","jest":"^28.1.1","pod-install":"^0.1.0","prettier":"^2.0.5","react":"18.2.0","react-native":"0.72.3","react-native-builder-bob":"^0.20.0","release-it":"^15.0.0","typedoc":"^0.24.7","typedoc-plugin-markdown":"^3.15.3","typescript":"^5.0.2"},"resolutions":{"@types/react":"17.0.21"},"peerDependencies":{"react":"*","react-native":"*"},"engines":{"node":">= 16.0.0"},"jest":{"preset":"react-native","modulePathIgnorePatterns":["<rootDir>/example/node_modules","<rootDir>/lib/"]},"commitlint":{"extends":["@commitlint/config-conventional"]},"release-it":{"git":{"commitMessage":"chore: release ${version}","tagName":"v${version}","requireCleanWorkingDir":false},"npm":{"publish":true,"skipChecks":true,"access":"public"},"github":{"release":false},"plugins":{"@release-it/conventional-changelog":{"preset":"angular"}}},"eslintConfig":{"extends":"@fugood/eslint-config-react","env":{"browser":true,"jest":true},"globals":{},"rules":{"react/no-array-index-key":0,"global-require":0},"overrides":[{"files":["*.ts","*.tsx"],"parser":"@typescript-eslint/parser","plugins":["@typescript-eslint/eslint-plugin"],"rules":{"@typescript-eslint/no-unused-vars":["error",{"argsIgnorePattern":"^_","destructuredArrayIgnorePattern":"^_"}],"no-unused-vars":"off","no-shadow":"off","@typescript-eslint/no-shadow":1,"no-undef":"off","func-call-spacing":"off","@typescript-eslint/func-call-spacing":1,"import/extensions":["error","ignorePackages",{"js":"never","jsx":"never","ts":"never","tsx":"never"}],"react/jsx-filename-extension":[1,{"extensions":[".js",".jsx",".tsx"]}]}}],"settings":{"import/resolver":{"node":{"extensions":[".js",".jsx",".ts",".tsx"]}}}},"eslintIgnore":["node_modules/","lib/","*.d.ts","llama.cpp/"],"prettier":{"trailingComma":"all","tabWidth":2,"semi":false,"singleQuote":true,"printWidth":80},"react-native-builder-bob":{"source":"src","output":"lib","targets":["commonjs","module",["typescript",{"project":"tsconfig.build.json"}]]},"codegenConfig":{"name":"RNLlamaSpec","type":"modules","jsSrcsDir":"src"},"packageManager":"yarn@1.22.22","_id":"@aghorbani/llama.rn@0.3.12","gitHead":"ed33a5e6cb813f0af62a39cd382caeded27c98fd","_nodeVersion":"20.18.0","_npmVersion":"10.8.2","dist":{"integrity":"sha512-VHuMVNkeX50R+er7Tyl/eiskI1tiddB/5WOSqVhyeuQHrjg3BCR1vlNczN2COiOBFgsVFVI9naraFh4xAyXJ+w==","shasum":"02f041ec1a2926d0a9a992e9a69efcdc83f4abfd","tarball":"https://registry.npmjs.org/@aghorbani/llama.rn/-/llama.rn-0.3.12.tgz","fileCount":92,"unpackedSize":7131428,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCgdVEabA+aLmT48uafoPkbPKcZ549cT4farhNUUSm4sQIgM6OeB+WqwACaDRE4E6z8s53MtqHR+QrXxZ6NCRoP13c="}]},"_npmUser":{"name":"aghorbani","email":"ghorbani59@gmail.com"},"directories":{},"maintainers":[{"name":"aghorbani","email":"ghorbani59@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/llama.rn_0.3.12_1730449500170_0.3221310942693161"},"_hasShrinkwrap":false}},"time":{"created":"2024-11-01T08:25:00.040Z","0.3.12":"2024-11-01T08:25:00.487Z","modified":"2024-11-01T08:25:00.759Z"},"maintainers":[{"name":"aghorbani","email":"ghorbani59@gmail.com"}],"description":"React Native binding of llama.cpp","homepage":"https://github.com/mybigday/llama.rn#readme","keywords":["react-native","ios","android","large language model","LLM","Local LLM","llama.cpp","llama","llama-2"],"repository":{"type":"git","url":"git+https://github.com/mybigday/llama.rn.git"},"author":{"name":"Jhen-Jie Hong","email":"developer@jhen.me","url":"https://github.com/mybigday"},"bugs":{"url":"https://github.com/mybigday/llama.rn/issues"},"license":"MIT","readme":"# llama.rn\n\n[![Actions Status](https://github.com/mybigday/llama.rn/workflows/CI/badge.svg)](https://github.com/mybigday/llama.rn/actions)\n[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)\n[![npm](https://img.shields.io/npm/v/llama.rn.svg)](https://www.npmjs.com/package/llama.rn/)\n\nReact Native binding of [llama.cpp](https://github.com/ggerganov/llama.cpp).\n\n[llama.cpp](https://github.com/ggerganov/llama.cpp): Inference of [LLaMA](https://arxiv.org/abs/2302.13971) model in pure C/C++\n\n## Installation\n\n```sh\nnpm install llama.rn\n```\n\n#### iOS\n\nPlease re-run `npx pod-install` again.\n\n#### Android\n\nAdd proguard rule if it's enabled in project (android/app/proguard-rules.pro):\n\n```proguard\n# llama.rn\n-keep class com.rnllama.** { *; }\n```\n\n## Obtain the model\n\nYou can search HuggingFace for available models (Keyword: [`GGUF`](https://huggingface.co/search/full-text?q=GGUF&type=model)).\n\nFor get a GGUF model or quantize manually, see [`Prepare and Quantize`](https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#prepare-and-quantize) section in llama.cpp.\n\n## Usage\n\n```js\nimport { initLlama } from 'llama.rn'\n\n// Initial a Llama context with the model (may take a while)\nconst context = await initLlama({\n  model: 'file://<path to gguf model>',\n  use_mlock: true,\n  n_ctx: 2048,\n  n_gpu_layers: 1, // > 0: enable Metal on iOS\n  // embedding: true, // use embedding\n})\n\nconst stopWords = ['</s>', '<|end|>', '<|eot_id|>', '<|end_of_text|>', '<|im_end|>', '<|EOT|>', '<|END_OF_TURN_TOKEN|>', '<|end_of_turn|>', '<|endoftext|>']\n\n// Do chat completion\nconst msgResult = await context.completion(\n  {\n    messages: [\n      {\n        role: 'system',\n        content: 'This is a conversation between user and assistant, a friendly chatbot.',\n      },\n      {\n        role: 'user',\n        content: 'Hello!',\n      },\n    ],\n    n_predict: 100,\n    stop: stopWords,\n    // ...other params\n  },\n  (data) => {\n    // This is a partial completion callback\n    const { token } = data\n  },\n)\nconsole.log('Result:', msgResult.text)\nconsole.log('Timings:', msgResult.timings)\n\n// Or do text completion\nconst textResult = await context.completion(\n  {\n    prompt: 'This is a conversation between user and llama, a friendly chatbot. respond in simple markdown.\\n\\nUser: Hello!\\nLlama:',\n    n_predict: 100,\n    stop: [...stopWords, 'Llama:', 'User:'],\n    // ...other params\n  },\n  (data) => {\n    // This is a partial completion callback\n    const { token } = data\n  },\n)\nconsole.log('Result:', textResult.text)\nconsole.log('Timings:', textResult.timings)\n```\n\nThe binding’s deisgn inspired by [server.cpp](https://github.com/ggerganov/llama.cpp/tree/master/examples/server) example in llama.cpp, so you can map its API to LlamaContext:\n\n- `/completion` and `/chat/completions`: `context.completion(params, partialCompletionCallback)`\n- `/tokenize`: `context.tokenize(content)`\n- `/detokenize`: `context.detokenize(tokens)`\n- `/embedding`: `context.embedding(content)`\n- Other methods\n  - `context.loadSession(path)`\n  - `context.saveSession(path)`\n  - `context.stopCompletion()`\n  - `context.release()`\n\nPlease visit the [Documentation](docs/API) for more details.\n\nYou can also visit the [example](example) to see how to use it.\n\nRun the example:\n\n```bash\nyarn && yarn bootstrap\n\n# iOS\nyarn example ios\n# Use device\nyarn example ios --device \"<device name>\"\n# With release mode\nyarn example ios --mode Release\n\n# Android\nyarn example android\n# With release mode\nyarn example android --mode release\n```\n\nThis example used [react-native-document-picker](https://github.com/rnmods/react-native-document-picker) for select model.\n\n- iOS: You can move the model to iOS Simulator, or iCloud for real device.\n- Android: Selected file will be copied or downloaded to cache directory so it may be slow.\n\n## Grammar Sampling\n\nGBNF (GGML BNF) is a format for defining [formal grammars](https://en.wikipedia.org/wiki/Formal_grammar) to constrain model outputs in `llama.cpp`. For example, you can use it to force the model to generate valid JSON, or speak only in emojis.\n\nYou can see [GBNF Guide](https://github.com/ggerganov/llama.cpp/tree/master/grammars) for more details.\n\n`llama.rn` provided a built-in function to convert JSON Schema to GBNF:\n\n```js\nimport { initLlama, convertJsonSchemaToGrammar } from 'llama.rn'\n\nconst schema = {\n  /* JSON Schema, see below */\n}\n\nconst context = await initLlama({\n  model: 'file://<path to gguf model>',\n  use_mlock: true,\n  n_ctx: 2048,\n  n_gpu_layers: 1, // > 0: enable Metal on iOS\n  // embedding: true, // use embedding\n  grammar: convertJsonSchemaToGrammar({\n    schema,\n    propOrder: { function: 0, arguments: 1 },\n  }),\n})\n\nconst { text } = await context.completion({\n  prompt: 'Schedule a birthday party on Aug 14th 2023 at 8pm.',\n})\nconsole.log('Result:', text)\n// Example output:\n// {\"function\": \"create_event\",\"arguments\":{\"date\": \"Aug 14th 2023\", \"time\": \"8pm\", \"title\": \"Birthday Party\"}}\n```\n\n<details>\n<summary>JSON Schema example (Define function get_current_weather / create_event / image_search)</summary>\n\n```json5\n{\n  oneOf: [\n    {\n      type: 'object',\n      name: 'get_current_weather',\n      description: 'Get the current weather in a given location',\n      properties: {\n        function: {\n          const: 'get_current_weather',\n        },\n        arguments: {\n          type: 'object',\n          properties: {\n            location: {\n              type: 'string',\n              description: 'The city and state, e.g. San Francisco, CA',\n            },\n            unit: {\n              type: 'string',\n              enum: ['celsius', 'fahrenheit'],\n            },\n          },\n          required: ['location'],\n        },\n      },\n    },\n    {\n      type: 'object',\n      name: 'create_event',\n      description: 'Create a calendar event',\n      properties: {\n        function: {\n          const: 'create_event',\n        },\n        arguments: {\n          type: 'object',\n          properties: {\n            title: {\n              type: 'string',\n              description: 'The title of the event',\n            },\n            date: {\n              type: 'string',\n              description: 'The date of the event',\n            },\n            time: {\n              type: 'string',\n              description: 'The time of the event',\n            },\n          },\n          required: ['title', 'date', 'time'],\n        },\n      },\n    },\n    {\n      type: 'object',\n      name: 'image_search',\n      description: 'Search for an image',\n      properties: {\n        function: {\n          const: 'image_search',\n        },\n        arguments: {\n          type: 'object',\n          properties: {\n            query: {\n              type: 'string',\n              description: 'The search query',\n            },\n          },\n          required: ['query'],\n        },\n      },\n    },\n  ],\n}\n```\n\n</details>\n\n<details>\n<summary>Converted GBNF looks like</summary>\n\n```bnf\nspace ::= \" \"?\n0-function ::= \"\\\"get_current_weather\\\"\"\nstring ::=  \"\\\"\" (\n        [^\"\\\\] |\n        \"\\\\\" ([\"\\\\/bfnrt] | \"u\" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])\n      )* \"\\\"\" space\n0-arguments-unit ::= \"\\\"celsius\\\"\" | \"\\\"fahrenheit\\\"\"\n0-arguments ::= \"{\" space \"\\\"location\\\"\" space \":\" space string \",\" space \"\\\"unit\\\"\" space \":\" space 0-arguments-unit \"}\" space\n0 ::= \"{\" space \"\\\"function\\\"\" space \":\" space 0-function \",\" space \"\\\"arguments\\\"\" space \":\" space 0-arguments \"}\" space\n1-function ::= \"\\\"create_event\\\"\"\n1-arguments ::= \"{\" space \"\\\"date\\\"\" space \":\" space string \",\" space \"\\\"time\\\"\" space \":\" space string \",\" space \"\\\"title\\\"\" space \":\" space string \"}\" space\n1 ::= \"{\" space \"\\\"function\\\"\" space \":\" space 1-function \",\" space \"\\\"arguments\\\"\" space \":\" space 1-arguments \"}\" space\n2-function ::= \"\\\"image_search\\\"\"\n2-arguments ::= \"{\" space \"\\\"query\\\"\" space \":\" space string \"}\" space\n2 ::= \"{\" space \"\\\"function\\\"\" space \":\" space 2-function \",\" space \"\\\"arguments\\\"\" space \":\" space 2-arguments \"}\" space\nroot ::= 0 | 1 | 2\n```\n\n</details>\n\n## Mock `llama.rn`\n\nWe have provided a mock version of `llama.rn` for testing purpose you can use on Jest:\n\n```js\njest.mock('llama.rn', () => require('llama.rn/jest/mock'))\n```\n\n## NOTE\n\niOS:\n\n- The [Extended Virtual Addressing](https://developer.apple.com/documentation/bundleresources/entitlements/com_apple_developer_kernel_extended-virtual-addressing) capability is recommended to enable on iOS project.\n- Metal:\n  - We have tested to know some devices is not able to use Metal ('params.n_gpu_layers > 0') due to llama.cpp used SIMD-scoped operation, you can check if your device is supported in [Metal feature set tables](https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf), Apple7 GPU will be the minimum requirement.\n  - It's also not supported in iOS simulator due to [this limitation](https://developer.apple.com/documentation/metal/developing_metal_apps_that_run_in_simulator#3241609), we used constant buffers more than 14.\n\nAndroid:\n\n- Currently only supported arm64-v8a / x86_64 platform, this means you can't initialize a context on another platforms. The 64-bit platform are recommended because it can allocate more memory for the model.\n- No integrated any GPU backend yet.\n\n## Contributing\n\nSee the [contributing guide](CONTRIBUTING.md) to learn how to contribute to the repository and the development workflow.\n\n## License\n\nMIT\n\n---\n\nMade with [create-react-native-library](https://github.com/callstack/react-native-builder-bob)\n\n---\n\n<p align=\"center\">\n  <a href=\"https://bricks.tools\">\n    <img width=\"90px\" src=\"https://avatars.githubusercontent.com/u/17320237?s=200&v=4\">\n  </a>\n  <p align=\"center\">\n    Built and maintained by <a href=\"https://bricks.tools\">BRICKS</a>.\n  </p>\n</p>\n","readmeFilename":"README.md"}