{
  "_id": "6a44d4fb6ff6f2357551c247",
  "Package": "robotstxt",
  "Type": "Package",
  "Title": "A 'robots.txt' Parser and 'Webbot'/'Spider'/'Crawler'\nPermissions Checker",
  "Version": "0.7.15.9000",
  "Authors@R": "c(\nperson(\n\"Pedro\", \"Baltazar\", role = c(\"ctb\"),\nemail = \"pedrobtz@gmail.com\"\n),\nperson(\n\"Jordan\", \"Bradford\", role = c(\"cre\"),\nemail = \"jrdnbradford@gmail.com\"\n),\nperson(\n\"Peter\", \"Meissner\", role = c(\"aut\"),\nemail = \"retep.meissner@gmail.com\"\n),\nperson(\n\"Kun\", \"Ren\", email = \"mail@renkun.me\", role = c(\"aut\", \"cph\"),\ncomment = \"Author and copyright holder of list_merge.R.\"\n),\nperson(\"Oliver\", \"Keys\", role = \"ctb\", comment = \"original release code review\"),\nperson(\"Rich\", \"Fitz John\", role = \"ctb\", comment = \"original release code review\")\n)",
  "Description": "Provides functions to download and parse 'robots.txt'\nfiles. Ultimately the package makes it easy to check if bots\n(spiders, crawler, scrapers, ...) are allowed to access\nspecific resources on a domain.",
  "License": "MIT + file LICENSE",
  "BugReports": "https://github.com/ropensci/robotstxt/issues",
  "URL": "https://docs.ropensci.org/robotstxt/,\nhttps://github.com/ropensci/robotstxt",
  "VignetteBuilder": "knitr",
  "RoxygenNote": "7.3.2",
  "Encoding": "UTF-8",
  "Config/testthat/edition": "3",
  "Config/pak/sysreqs": "libicu-dev libssl-dev",
  "Repository": "https://ropensci.r-universe.dev",
  "Date/Publication": "2024-11-15 19:32:37 UTC",
  "RemoteUrl": "https://github.com/ropensci/robotstxt",
  "RemoteRef": "main",
  "RemoteSha": "d3d0a4d525e87204010d10162b01a921b05a6149",
  "NeedsCompilation": "no",
  "Packaged": {
    "Date": "2026-07-01 08:27:13 UTC",
    "User": "root"
  },
  "Author": "Pedro Baltazar [ctb],\nJordan Bradford [cre],\nPeter Meissner [aut],\nKun Ren [aut, cph] (Author and copyright holder of list_merge.R.),\nOliver Keys [ctb] (original release code review),\nRich Fitz John [ctb] (original release code review)",
  "Maintainer": "Jordan Bradford <jrdnbradford@gmail.com>",
  "_user": "ropensci",
  "_type": "src",
  "_file": "robotstxt_0.7.15.9000.tar.gz",
  "_fileid": "https://r2.ropensci.org/ea4ccaa5b02355cf5d510be4983ab1b02239b62e2a00dd3a6ab89f80500cb37d",
  "_filesize": 271370,
  "_sha256": "ea4ccaa5b02355cf5d510be4983ab1b02239b62e2a00dd3a6ab89f80500cb37d",
  "_expires": "2026-10-09T08:51:06.000Z",
  "_created": "2026-07-01T08:27:13.000Z",
  "_published": "2026-07-01T08:51:07.196Z",
  "_jobs": [
    {
      "job": 84488353490,
      "time": 138,
      "config": "linux-devel-x86_64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "8004333827"
    },
    {
      "job": 84488353521,
      "time": 142,
      "config": "linux-release-x86_64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "8004334950"
    },
    {
      "job": 84488353474,
      "time": 103,
      "config": "macos-oldrel-arm64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "8004788378"
    },
    {
      "job": 84488353559,
      "time": 134,
      "config": "macos-release-arm64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "8004801306"
    },
    {
      "job": 84487453665,
      "time": 157,
      "config": "pkgdown",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "8004251484"
    },
    {
      "job": 84487453642,
      "time": 200,
      "config": "source",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "8004267484"
    },
    {
      "job": 84488353484,
      "time": 131,
      "config": "wasm-release",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "8004331121"
    },
    {
      "job": 84488353498,
      "time": 88,
      "config": "windows-devel",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "8004314031"
    },
    {
      "job": 84488353518,
      "time": 85,
      "config": "windows-oldrel",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "8004312912"
    },
    {
      "job": 84488353501,
      "time": 77,
      "config": "windows-release",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "8004309099"
    }
  ],
  "_host": "GitHub-Actions",
  "_buildurl": "https://github.com/r-universe/ropensci/actions/runs/28503713854",
  "_status": "success",
  "_upstream": "https://github.com/ropensci/robotstxt",
  "_commit": {
    "id": "d3d0a4d525e87204010d10162b01a921b05a6149",
    "author": "Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com>",
    "committer": "GitHub <noreply@github.com>",
    "message": "Merge pull request #94 from ropensci/simplify-get_robotstxt_http_get\n\nRewrite `get_robotstxt_http_get`",
    "time": 1731699157
  },
  "_maintainer": {
    "name": "Jordan Bradford",
    "email": "jrdnbradford@gmail.com",
    "login": "jrdnbradford",
    "uuid": 36420801
  },
  "_distro": "resolute",
  "_registered": true,
  "_dependencies": [
    {
      "package": "R",
      "version": ">= 3.0.0",
      "role": "Depends"
    },
    {
      "package": "stringr",
      "version": ">= 1.0.0",
      "role": "Imports"
    },
    {
      "package": "httr",
      "version": ">= 1.0.0",
      "role": "Imports"
    },
    {
      "package": "spiderbar",
      "version": ">= 0.2.0",
      "role": "Imports"
    },
    {
      "package": "future.apply",
      "version": ">= 1.0.0",
      "role": "Imports"
    },
    {
      "package": "magrittr",
      "role": "Imports"
    },
    {
      "package": "utils",
      "role": "Imports"
    },
    {
      "package": "knitr",
      "role": "Suggests"
    },
    {
      "package": "rmarkdown",
      "role": "Suggests"
    },
    {
      "package": "dplyr",
      "role": "Suggests"
    },
    {
      "package": "testthat",
      "version": ">= 3.0.0",
      "role": "Suggests"
    },
    {
      "package": "covr",
      "role": "Suggests"
    },
    {
      "package": "curl",
      "role": "Suggests"
    }
  ],
  "_owner": "ropensci",
  "_selfowned": true,
  "_usedby": 5,
  "_updates": [],
  "_tags": [],
  "_topics": [
    "crawler",
    "peer-reviewed",
    "robotstxt",
    "scraper",
    "spider",
    "webscraping"
  ],
  "_stars": 69,
  "_contributors": [
    {
      "user": "petermeissner",
      "count": 248,
      "uuid": 939139
    },
    {
      "user": "jrdnbradford",
      "count": 32,
      "uuid": 36420801
    },
    {
      "user": "pedrobtz",
      "count": 19,
      "uuid": 2864346
    },
    {
      "user": "dmi3kno",
      "count": 5,
      "uuid": 13419011
    },
    {
      "user": "gittaca",
      "count": 3,
      "uuid": 17674964
    },
    {
      "user": "maelle",
      "count": 2,
      "uuid": 8360597
    },
    {
      "user": "sckott",
      "count": 2,
      "uuid": 577668
    },
    {
      "user": "karthik",
      "count": 1,
      "uuid": 138494
    },
    {
      "user": "mine-cetinkaya-rundel",
      "count": 1,
      "uuid": 5965649
    }
  ],
  "_userbio": {
    "uuid": 1200269,
    "type": "organization",
    "name": "rOpenSci",
    "followers": 1106,
    "description": "Tools and R Packages for Open Science"
  },
  "_downloads": {
    "count": 1096,
    "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/robotstxt"
  },
  "_devurl": "https://github.com/ropensci/robotstxt",
  "_pkgdown": "https://docs.ropensci.org/robotstxt/",
  "_searchresults": 516,
  "_metadata": {
    "review": {
      "id": 25,
      "status": "reviewed",
      "version": "0.1.0",
      "organization": "rOpenSci Software Review",
      "url": "https://github.com/ropensci/software-review/issues/25"
    },
    "ropensci_category": "http-tools"
  },
  "_rbuild": "4.6.1",
  "_assets": [
    "extra/citation.cff",
    "extra/citation.html",
    "extra/citation.json",
    "extra/citation.txt",
    "extra/contents.json",
    "extra/NEWS.html",
    "extra/NEWS.txt",
    "extra/readme.html",
    "extra/readme.md",
    "extra/robotstxt.html",
    "LICENSE",
    "manual.pdf"
  ],
  "_homeurl": "https://github.com/ropensci/robotstxt",
  "_realowner": "ropensci",
  "_cranurl": true,
  "_releases": [
    {
      "version": "0.1.2",
      "date": "2016-02-08"
    },
    {
      "version": "0.3.2",
      "date": "2016-04-27"
    },
    {
      "version": "0.4.0",
      "date": "2017-07-16"
    },
    {
      "version": "0.4.1",
      "date": "2017-09-01"
    },
    {
      "version": "0.5.2",
      "date": "2017-11-12"
    },
    {
      "version": "0.6.0",
      "date": "2018-02-11"
    },
    {
      "version": "0.6.2",
      "date": "2018-07-18"
    },
    {
      "version": "0.7.4",
      "date": "2020-05-31"
    },
    {
      "version": "0.7.7",
      "date": "2020-06-27"
    },
    {
      "version": "0.7.8",
      "date": "2020-07-25"
    },
    {
      "version": "0.7.13",
      "date": "2020-09-03"
    },
    {
      "version": "0.7.15",
      "date": "2024-08-29"
    }
  ],
  "_exports": [
    "%>%",
    "get_robotstxt",
    "get_robotstxt_http_get",
    "get_robotstxts",
    "is_valid_robotstxt",
    "on_client_error_default",
    "on_domain_change_default",
    "on_file_type_mismatch_default",
    "on_not_found_default",
    "on_redirect_default",
    "on_server_error_default",
    "on_sub_domain_change_default",
    "on_suspect_content_default",
    "parse_robotstxt",
    "paths_allowed",
    "request_handler_handler",
    "robotstxt",
    "rt_last_http",
    "rt_request_handler"
  ],
  "_help": [
    {
      "page": "pipe",
      "title": "re-export magrittr pipe operator",
      "topics": [
        "%>%"
      ]
    },
    {
      "page": "as.list.robotstxt_text",
      "title": "Convert robotstxt_text to list",
      "topics": [
        "as.list.robotstxt_text"
      ]
    },
    {
      "page": "fix_url",
      "title": "Add http protocal if missing from URL",
      "topics": [
        "fix_url"
      ]
    },
    {
      "page": "get_robotstxt",
      "title": "Download a robots.txt file",
      "topics": [
        "get_robotstxt"
      ]
    },
    {
      "page": "get_robotstxts",
      "title": "Download multiple robotstxt files",
      "topics": [
        "get_robotstxts"
      ]
    },
    {
      "page": "guess_domain",
      "title": "Guess a domain from path",
      "topics": [
        "guess_domain"
      ]
    },
    {
      "page": "http_domain_changed",
      "title": "Check if HTTP domain changed",
      "topics": [
        "http_domain_changed"
      ]
    },
    {
      "page": "http_subdomain_changed",
      "title": "Check if HTTP subdomain changed",
      "topics": [
        "http_subdomain_changed"
      ]
    },
    {
      "page": "http_was_redirected",
      "title": "Check if HTTP redirect occurred",
      "topics": [
        "http_was_redirected"
      ]
    },
    {
      "page": "is_suspect_robotstxt",
      "title": "Check if file is valid / parsable robots.txt file",
      "topics": [
        "is_suspect_robotstxt"
      ]
    },
    {
      "page": "is_valid_robotstxt",
      "title": "Validate if a file is valid / parsable robots.txt file",
      "topics": [
        "is_valid_robotstxt"
      ]
    },
    {
      "page": "list_merge",
      "title": "Merge a number of named lists in sequential order",
      "topics": [
        "list_merge"
      ]
    },
    {
      "page": "null_to_default",
      "title": "Return default value if NULL",
      "topics": [
        "null_to_default"
      ]
    },
    {
      "page": "parse_robotstxt",
      "title": "Parse a robots.txt file",
      "topics": [
        "parse_robotstxt"
      ]
    },
    {
      "page": "paths_allowed",
      "title": "Check if a bot has permissions to access page(s)",
      "topics": [
        "paths_allowed"
      ]
    },
    {
      "page": "paths_allowed_worker_spiderbar",
      "title": "Check if a spiderbar bot has permissions to access page(s)",
      "topics": [
        "paths_allowed_worker_spiderbar"
      ]
    },
    {
      "page": "print.robotstxt",
      "title": "Print robotstxt",
      "topics": [
        "print.robotstxt"
      ]
    },
    {
      "page": "print.robotstxt_text",
      "title": "Print robotstxt's text",
      "topics": [
        "print.robotstxt_text"
      ]
    },
    {
      "page": "remove_domain",
      "title": "Remove domain from path",
      "topics": [
        "remove_domain"
      ]
    },
    {
      "page": "request_handler_handler",
      "title": "Handle robotstxt handlers",
      "topics": [
        "request_handler_handler"
      ]
    },
    {
      "page": "robotstxt",
      "title": "Generate a representation of a robots.txt file",
      "topics": [
        "robotstxt"
      ]
    },
    {
      "page": "rt_cache",
      "title": "Get the robotstxt cache",
      "topics": [
        "rt_cache"
      ]
    },
    {
      "page": "get_robotstxt_http_get",
      "title": "Storage for HTTP request response objects",
      "topics": [
        "get_robotstxt_http_get",
        "rt_last_http"
      ]
    },
    {
      "page": "rt_request_handler",
      "title": "Handle robotstxt object retrieved from HTTP request",
      "topics": [
        "on_client_error_default",
        "on_domain_change_default",
        "on_file_type_mismatch_default",
        "on_not_found_default",
        "on_redirect_default",
        "on_server_error_default",
        "on_sub_domain_change_default",
        "on_suspect_content_default",
        "rt_request_handler"
      ]
    }
  ],
  "_readme": "https://github.com/ropensci/robotstxt/raw/main/README.md",
  "_rundeps": [
    "askpass",
    "cli",
    "codetools",
    "curl",
    "digest",
    "future",
    "future.apply",
    "globals",
    "glue",
    "httr",
    "jsonlite",
    "lifecycle",
    "listenv",
    "magrittr",
    "mime",
    "openssl",
    "parallelly",
    "R6",
    "Rcpp",
    "rlang",
    "spiderbar",
    "stringi",
    "stringr",
    "sys",
    "vctrs"
  ],
  "_vignettes": [
    {
      "source": "using_robotstxt.Rmd",
      "filename": "using_robotstxt.html",
      "title": "Using Robotstxt",
      "author": "Peter Meissner",
      "engine": "knitr::rmarkdown",
      "headings": [
        "Description",
        "Robots.txt files",
        "Fast food usage for the uninterested",
        "Example Usage",
        "object oriented style",
        "functional style"
      ],
      "created": "2016-01-09 20:19:32",
      "modified": "2024-08-24 11:05:33",
      "commits": 9
    }
  ],
  "_score": 9.36946059689646,
  "_indexed": true,
  "_nocasepkg": "robotstxt",
  "_universes": [
    "ropensci",
    "jrdnbradford"
  ],
  "_binaries": [
    {
      "r": "4.7.0",
      "os": "linux",
      "version": "0.7.15.9000",
      "date": "2026-07-01T08:30:01.000Z",
      "distro": "resolute",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/77b7412b78ad6b9d98527342d4cc7f99f9e418a04dc80228e0cb2069607efd08",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/28503713854"
    },
    {
      "r": "4.6.1",
      "os": "linux",
      "version": "0.7.15.9000",
      "date": "2026-07-01T08:29:58.000Z",
      "distro": "resolute",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/a1d5651d8fdf566e63d46f0b4e7066270233688911db130c7e517117188169f8",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/28503713854"
    },
    {
      "r": "4.5.3",
      "os": "mac",
      "version": "0.7.15.9000",
      "date": "2026-07-01T08:49:50.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/62618d82a0769512bd3a1cb4b3af0ac7a187e85f4f97e5470505eced7fe207f9",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/28503713854"
    },
    {
      "r": "4.6.1",
      "os": "mac",
      "version": "0.7.15.9000",
      "date": "2026-07-01T08:50:10.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/7c48e5a49ac40d9feb3d4d234c2c501231d4717201683abe8e29de288b93a060",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/28503713854"
    },
    {
      "r": "4.6.0",
      "os": "wasm",
      "version": "0.7.15.9000",
      "date": "2026-07-01T08:30:10.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/3a7c29592200ff5b427eda6f7567ac9bd321a40141773740930f9824c793fd51",
      "status": "success",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/28503713854"
    },
    {
      "r": "4.7.0",
      "os": "win",
      "version": "0.7.15.9000",
      "date": "2026-07-01T08:29:02.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/b505d79e4f9c3f9aff2853509f00aa6df231e2e35854afbf8f9ae1e5b87e84bd",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/28503713854"
    },
    {
      "r": "4.5.3",
      "os": "win",
      "version": "0.7.15.9000",
      "date": "2026-07-01T08:28:57.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/2f714988d473a2971d588b51f644bb926d5f7dee397d02250879559739f6ef06",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/28503713854"
    },
    {
      "r": "4.6.1",
      "os": "win",
      "version": "0.7.15.9000",
      "date": "2026-07-01T08:28:51.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/ff2e2c40d167b6f48b762bbe9a13d747f72a1a344b4946e343d8a0d4ee7a64f8",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/28503713854"
    }
  ]
}