{
  "_id": "6ae473ec3d0f299392dd300e",
  "Package": "tokenizers",
  "Type": "Package",
  "Title": "Fast, Consistent Tokenization of Natural Language Text",
  "Version": "0.3.1",
  "Date": "2024-03-27",
  "Description": "Convert natural language text into tokens. Includes\ntokenizers for shingled n-grams, skip n-grams, words, word\nstems, sentences, paragraphs, characters, shingled characters,\nlines, Penn Treebank, regular expressions, as well as functions\nfor counting characters, words, and sentences, and a function\nfor splitting longer texts into separate documents, each with\nthe same number of words.  The tokenizers have a consistent\ninterface, and the package is built on the 'stringi' and 'Rcpp'\npackages for fast yet correct tokenization in 'UTF-8'.",
  "License": "MIT + file LICENSE",
  "LazyData": "yes",
  "Authors@R": "c(person(\"Thomas\", \"Charlon\", role = c(\"aut\", \"cre\"),\nemail = \"charlon@protonmail.com\",\ncomment = c(ORCID = \"0000-0001-7497-0470\")),\nperson(\"Lincoln\", \"Mullen\", role = c(\"aut\"),\nemail = \"lincoln@lincolnmullen.com\",\ncomment = c(ORCID = \"0000-0001-5103-6917\")),\nperson(\"Os\", \"Keyes\", role = c(\"ctb\"),\nemail = \"ironholds@gmail.com\",\ncomment = c(ORCID = \"0000-0001-5196-609X\")),\nperson(\"Dmitriy\", \"Selivanov\", role = c(\"ctb\"),\nemail = \"selivanov.dmitriy@gmail.com\"),\nperson(\"Jeffrey\", \"Arnold\", role = c(\"ctb\"),\nemail = \"jeffrey.arnold@gmail.com\",\ncomment = c(ORCID = \"0000-0001-9953-3904\")),\nperson(\"Kenneth\", \"Benoit\", role = c(\"ctb\"),\nemail = \"kbenoit@lse.ac.uk\",\ncomment = c(ORCID = \"0000-0002-0797-564X\")))",
  "URL": "https://docs.ropensci.org/tokenizers/,\nhttps://github.com/ropensci/tokenizers",
  "BugReports": "https://github.com/ropensci/tokenizers/issues",
  "RoxygenNote": "7.3.1",
  "Encoding": "UTF-8",
  "VignetteBuilder": "knitr",
  "Config/pak/sysreqs": "libicu-dev",
  "Repository": "https://ropensci.r-universe.dev",
  "Date/Publication": "2024-03-27 09:33:34 UTC",
  "RemoteUrl": "https://github.com/ropensci/tokenizers",
  "RemoteRef": "HEAD",
  "RemoteSha": "b80863d088d4b39695b602ca11e061ac34770ec7",
  "NeedsCompilation": "yes",
  "Packaged": {
    "Date": "2026-10-30 09:55:42 UTC",
    "User": "root"
  },
  "Author": "Thomas Charlon [aut, cre] (ORCID:\n<https://orcid.org/0000-0001-7497-0470>),\nLincoln Mullen [aut] (ORCID: <https://orcid.org/0000-0001-5103-6917>),\nOs Keyes [ctb] (ORCID: <https://orcid.org/0000-0001-5196-609X>),\nDmitriy Selivanov [ctb],\nJeffrey Arnold [ctb] (ORCID: <https://orcid.org/0000-0001-9953-3904>),\nKenneth Benoit [ctb] (ORCID: <https://orcid.org/0000-0002-0797-564X>)",
  "Maintainer": "Thomas Charlon <charlon@protonmail.com>",
  "_user": "ropensci",
  "_type": "src",
  "_file": "tokenizers_0.3.1.tar.gz",
  "_fileid": "https://r2.ropensci.org/ec58d79260a5f37a745728b226c666b16313dfffce79fc680f1e9e0f714c05f0",
  "_filesize": 574295,
  "_sha256": "ec58d79260a5f37a745728b226c666b16313dfffce79fc680f1e9e0f714c05f0",
  "_expires": "2027-02-07T10:39:39.000Z",
  "_created": "2026-10-30T09:55:42.000Z",
  "_published": "2026-10-30T10:39:40.866Z",
  "_upstream": "https://github.com/ropensci/tokenizers",
  "_status": "success",
  "_maintainer": {
    "name": "Thomas Charlon",
    "email": "charlon@protonmail.com",
    "login": "thomaschln",
    "linkedin": "in/thomas-charlon-meng-phd-aba0a3275",
    "orcid": "0000-0001-7497-0470",
    "description": "Harvard Medical School Researcher\nBiomedical Informatics @hms-dbmi\nCELEHS laboratory @celehs",
    "uuid": 2394508
  },
  "_jobs": [
    {
      "job": 124000248531,
      "time": 100,
      "config": "linux-devel-arm64",
      "r": "4.7.0",
      "check": "NOTE",
      "artifact": "12968746319"
    },
    {
      "job": 124000248347,
      "time": 113,
      "config": "linux-devel-x86_64",
      "r": "4.7.0",
      "check": "NOTE",
      "artifact": "12967214840"
    },
    {
      "job": 124000248337,
      "time": 101,
      "config": "linux-release-arm64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "12968258581"
    },
    {
      "job": 124000248669,
      "time": 127,
      "config": "linux-release-x86_64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "12968557760"
    },
    {
      "job": 124000248591,
      "time": 71,
      "config": "macos-oldrel-arm64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "12971235785"
    },
    {
      "job": 124000248341,
      "time": 201,
      "config": "macos-oldrel-x86_64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "12968184251"
    },
    {
      "job": 124000248382,
      "time": 113,
      "config": "macos-release-arm64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "12969669789"
    },
    {
      "job": 124000248378,
      "time": 270,
      "config": "macos-release-x86_64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "12968413654"
    },
    {
      "job": 123999028886,
      "time": 166,
      "config": "pkgdown",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "12967134897"
    },
    {
      "job": 123999028970,
      "time": 182,
      "config": "source",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "12968103712"
    },
    {
      "job": 124000248366,
      "time": 106,
      "config": "wasm-release",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "12967719852"
    },
    {
      "job": 124000248483,
      "time": 226,
      "config": "windows-devel-arm64",
      "r": "4.7.0",
      "check": "NOTE",
      "artifact": "12968925801"
    },
    {
      "job": 124000248361,
      "time": 145,
      "config": "windows-devel-x86_64",
      "r": "4.7.0",
      "check": "NOTE",
      "artifact": "12967369537"
    },
    {
      "job": 124000248656,
      "time": 115,
      "config": "windows-oldrel-x86_64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "12967489089"
    },
    {
      "job": 124000248371,
      "time": 222,
      "config": "windows-release-arm64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "12968752025"
    },
    {
      "job": 124000248505,
      "time": 105,
      "config": "windows-release-x86_64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "12968860582"
    }
  ],
  "_buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422",
  "_commit": {
    "id": "b80863d088d4b39695b602ca11e061ac34770ec7",
    "author": "tcharlon <charlon@protonmail.com>",
    "committer": "tcharlon <charlon@protonmail.com>",
    "message": "new maintainer\n",
    "time": 1711532014
  },
  "_distro": "resolute",
  "_host": "GitHub-Actions",
  "_registered": true,
  "_dependencies": [
    {
      "package": "R",
      "version": ">= 3.1.3",
      "role": "Depends"
    },
    {
      "package": "Rcpp",
      "role": "LinkingTo"
    },
    {
      "package": "stringi",
      "version": ">= 1.0.1",
      "role": "Imports"
    },
    {
      "package": "Rcpp",
      "version": ">= 0.12.3",
      "role": "Imports"
    },
    {
      "package": "SnowballC",
      "version": ">= 0.5.1",
      "role": "Imports"
    },
    {
      "package": "covr",
      "role": "Suggests"
    },
    {
      "package": "knitr",
      "role": "Suggests"
    },
    {
      "package": "rmarkdown",
      "role": "Suggests"
    },
    {
      "package": "stopwords",
      "version": ">= 0.9.0",
      "role": "Suggests"
    },
    {
      "package": "testthat",
      "role": "Suggests"
    }
  ],
  "_owner": "ropensci",
  "_selfowned": true,
  "_usedby": 79,
  "_updates": [],
  "_tags": [],
  "_topics": [
    "nlp",
    "peer-reviewed",
    "text-mining",
    "tokenizer",
    "cpp"
  ],
  "_stars": 188,
  "_contributors": [
    {
      "user": "lmullen",
      "count": 175,
      "uuid": 183672
    },
    {
      "user": "dselivanov",
      "count": 6,
      "uuid": 5123805
    },
    {
      "user": "kbenoit",
      "count": 4,
      "uuid": 2182246
    },
    {
      "user": "jrnold",
      "count": 4,
      "uuid": 123968
    },
    {
      "user": "chrismuir",
      "count": 1,
      "uuid": 13386824
    },
    {
      "user": "emilhvitfeldt",
      "count": 1,
      "uuid": 14034784
    },
    {
      "user": "hideaki",
      "count": 1,
      "uuid": 19518
    },
    {
      "user": "jeroen",
      "count": 1,
      "uuid": 216319
    },
    {
      "user": "juliasilge",
      "count": 1,
      "uuid": 12505835
    },
    {
      "user": "karthik",
      "count": 1,
      "uuid": 138494
    },
    {
      "user": "maelle",
      "count": 1,
      "uuid": 8360597
    },
    {
      "user": "ironholds",
      "count": 1,
      "uuid": 2487262
    },
    {
      "user": "thomaschln",
      "count": 1,
      "uuid": 2394508
    }
  ],
  "_userbio": {
    "uuid": 1200269,
    "type": "organization",
    "name": "rOpenSci",
    "followers": 1166,
    "description": "Tools and R Packages for Open Science"
  },
  "_downloads": {
    "count": 34867,
    "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/tokenizers"
  },
  "_mentions": 1,
  "_devurl": "https://github.com/ropensci/tokenizers",
  "_pkgdown": "https://docs.ropensci.org/tokenizers/",
  "_searchresults": 22,
  "_metadata": {
    "review": {
      "id": 33,
      "status": "reviewed",
      "version": "0.1.1",
      "organization": "rOpenSci Software Review",
      "url": "https://github.com/ropensci/software-review/issues/33"
    },
    "ropensci_category": "scalereprod"
  },
  "_rbuild": "4.6.1",
  "_assets": [
    "extra/citation.cff",
    "extra/citation.html",
    "extra/citation.json",
    "extra/citation.txt",
    "extra/contents.json",
    "extra/manual.pdf",
    "extra/NEWS.html",
    "extra/NEWS.txt",
    "extra/readme.html",
    "extra/readme.md",
    "extra/tokenizers.html",
    "LICENSE"
  ],
  "_homeurl": "https://github.com/ropensci/tokenizers",
  "_realowner": "ropensci",
  "_cranurl": true,
  "_releases": [
    {
      "version": "0.1.0",
      "date": "2016-04-02"
    },
    {
      "version": "0.1.1",
      "date": "2016-04-04"
    },
    {
      "version": "0.1.2",
      "date": "2016-04-14"
    },
    {
      "version": "0.1.3",
      "date": "2016-08-18"
    },
    {
      "version": "0.1.4",
      "date": "2016-08-29"
    },
    {
      "version": "0.2.0",
      "date": "2018-03-21"
    },
    {
      "version": "0.2.1",
      "date": "2018-03-29"
    },
    {
      "version": "0.2.3",
      "date": "2022-09-23"
    },
    {
      "version": "0.3.0",
      "date": "2022-12-22"
    }
  ],
  "_exports": [
    "chunk_text",
    "count_characters",
    "count_sentences",
    "count_words",
    "tokenize_character_shingles",
    "tokenize_characters",
    "tokenize_lines",
    "tokenize_ngrams",
    "tokenize_paragraphs",
    "tokenize_ptb",
    "tokenize_regex",
    "tokenize_sentences",
    "tokenize_skip_ngrams",
    "tokenize_word_stems",
    "tokenize_words"
  ],
  "_datasets": [
    {
      "name": "mobydick",
      "title": "The text of Moby Dick",
      "object": "mobydick",
      "class": [
        "character"
      ],
      "fields": [],
      "table": false,
      "tojson": true
    }
  ],
  "_help": [
    {
      "page": "basic-tokenizers",
      "title": "Basic tokenizers",
      "topics": [
        "basic-tokenizers",
        "tokenize_characters",
        "tokenize_lines",
        "tokenize_paragraphs",
        "tokenize_regex",
        "tokenize_sentences",
        "tokenize_words"
      ]
    },
    {
      "page": "chunk_text",
      "title": "Chunk text into smaller segments",
      "topics": [
        "chunk_text"
      ]
    },
    {
      "page": "word-counting",
      "title": "Count words, sentences, characters",
      "topics": [
        "count_characters",
        "count_sentences",
        "count_words"
      ]
    },
    {
      "page": "mobydick",
      "title": "The text of Moby Dick",
      "topics": [
        "mobydick"
      ]
    },
    {
      "page": "ngram-tokenizers",
      "title": "N-gram tokenizers",
      "topics": [
        "ngram-tokenizers",
        "tokenize_ngrams",
        "tokenize_skip_ngrams"
      ]
    },
    {
      "page": "shingle-tokenizers",
      "title": "Character shingle tokenizers",
      "topics": [
        "tokenize_character_shingles"
      ]
    },
    {
      "page": "ptb-tokenizer",
      "title": "Penn Treebank Tokenizer",
      "topics": [
        "tokenize_ptb"
      ]
    },
    {
      "page": "stem-tokenizers",
      "title": "Word stem tokenizer",
      "topics": [
        "tokenize_word_stems"
      ]
    },
    {
      "page": "tokenizers",
      "title": "Tokenizers",
      "topics": [
        "tokenizers-package",
        "tokenizers"
      ]
    }
  ],
  "_readme": "https://github.com/ropensci/tokenizers/raw/HEAD/README.md",
  "_rundeps": [
    "Rcpp",
    "SnowballC",
    "stringi"
  ],
  "_sysdeps": [
    {
      "shlib": "libstdc++",
      "package": "libstdc++6",
      "source": "gcc",
      "version": "16-20260322-1ubuntu1",
      "name": "c++",
      "homepage": "http://gcc.gnu.org/",
      "description": "GNU Standard C++ Library v3"
    }
  ],
  "_vignettes": [
    {
      "source": "introduction-to-tokenizers.Rmd",
      "filename": "introduction-to-tokenizers.html",
      "title": "Introduction to the tokenizers Package",
      "author": "Lincoln Mullen",
      "engine": "knitr::rmarkdown",
      "headings": [
        "Package overview",
        "Character and character-shingle tokenizers",
        "Word and word-stem tokenizers",
        "N-gram and skip n-gram tokenizers",
        "Sentence and paragraph tokenizers",
        "Text chunking",
        "Counting words, characters, sentences"
      ],
      "created": "2016-08-11 20:12:37",
      "modified": "2022-12-19 21:14:10",
      "commits": 12
    },
    {
      "source": "tif-and-tokenizers.Rmd",
      "filename": "tif-and-tokenizers.html",
      "title": "The Text Interchange Formats and the tokenizers Package",
      "author": "Lincoln Mullen",
      "engine": "knitr::rmarkdown",
      "headings": [],
      "created": "2018-03-14 00:10:35",
      "modified": "2022-09-23 18:07:51",
      "commits": 5
    }
  ],
  "_score": 11.612924703890254,
  "_indexed": true,
  "_nocasepkg": "tokenizers",
  "_universes": [
    "ropensci",
    "thomaschln"
  ],
  "_binaries": [
    {
      "r": "4.7.0",
      "os": "linux",
      "version": "0.3.1",
      "date": "2026-10-30T09:57:26.000Z",
      "distro": "resolute",
      "arch": "aarch64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/ce37915f48b8fdd0df85f920b2a0b921c151d6e564b6aeb70022039edc30c2b0",
      "status": "success",
      "check": "NOTE",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.7.0",
      "os": "linux",
      "version": "0.3.1",
      "date": "2026-10-30T09:57:39.000Z",
      "distro": "resolute",
      "arch": "x86_64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/d1234b25776b0ae0e24a720336850afc2ad247a83ac8b6f9867dd351392c8547",
      "status": "success",
      "check": "NOTE",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.6.1",
      "os": "linux",
      "version": "0.3.1",
      "date": "2026-10-30T09:57:29.000Z",
      "distro": "resolute",
      "arch": "aarch64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/11acc82e91813dc9964a5edde47d2875c8bfc8d91517e59457a487172eb6e2d0",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.6.1",
      "os": "linux",
      "version": "0.3.1",
      "date": "2026-10-30T09:57:46.000Z",
      "distro": "resolute",
      "arch": "x86_64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/765e9124bacbacdfcd05ef1fb29038d99bb4ec79926487fb8d3fbc1ee77a8871",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.5.3",
      "os": "mac",
      "version": "0.3.1",
      "date": "2026-10-30T10:38:16.000Z",
      "arch": "aarch64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/1cadccde7e29f760b3f65d5e9b0908c49bd71271de59ea01be78f7cc320721fe",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.5.3",
      "os": "mac",
      "version": "0.3.1",
      "date": "2026-10-30T09:58:36.000Z",
      "arch": "x86_64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/63a3cc793c0484c41fc9ce2f4591febaed7b6a9e8e40546eb2bc23d14247d644",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.6.1",
      "os": "mac",
      "version": "0.3.1",
      "date": "2026-10-30T10:38:45.000Z",
      "arch": "aarch64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/5702a292b98e626b9dc3e804cb269bd8d603be47d30ec5447d50718732deb425",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.6.1",
      "os": "mac",
      "version": "0.3.1",
      "date": "2026-10-30T09:59:28.000Z",
      "arch": "x86_64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/5f73f7a62cf53aaf96ac957a9173a4fd223a088a3ba825dcb7f2f0b741706465",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.6.0",
      "os": "wasm",
      "version": "0.3.1",
      "date": "2026-10-30T09:57:41.000Z",
      "arch": "emscripten",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/321623b8786edb8147e4198994e8e9c57d9677f334546effa46fe2ac4392ce3c",
      "status": "success",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.7.0",
      "os": "win",
      "version": "0.3.1",
      "date": "2026-10-30T09:59:14.000Z",
      "arch": "aarch64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/78daff653f905bacb22e7f578e33dfb45e1d486d6fe93d9f8fb5cff2ceceb354",
      "status": "success",
      "check": "NOTE",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.7.0",
      "os": "win",
      "version": "0.3.1",
      "date": "2026-10-30T09:57:00.000Z",
      "arch": "x86_64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/cface2ebb884d95ec8bbb21de4635b829cbef4281e705adf6e8e5dc63bc654ed",
      "status": "success",
      "check": "NOTE",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.5.3",
      "os": "win",
      "version": "0.3.1",
      "date": "2026-10-30T09:57:13.000Z",
      "arch": "x86_64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/0611ad1857214c7bfe805ecd81a0a35159dd4616a55aa200de1f4f89fcd97ee4",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.6.1",
      "os": "win",
      "version": "0.3.1",
      "date": "2026-10-30T09:59:12.000Z",
      "arch": "aarch64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/342caf9084c9cae82553880a668df2a0802aeb2dbcfa11e51ae490235354ba8f",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    },
    {
      "r": "4.6.1",
      "os": "win",
      "version": "0.3.1",
      "date": "2026-10-30T09:57:03.000Z",
      "arch": "x86_64",
      "commit": "b80863d088d4b39695b602ca11e061ac34770ec7",
      "fileid": "https://r2.ropensci.org/6d2a5d30ee004a7a115c6bfda00a1b90b357ade22396acc5fccf17e1500b1a13",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/41036609422"
    }
  ]
}