{
  "_id": "6ae300ca3d0f299392dd06d5",
  "Package": "robotstxt",
  "Type": "Package",
  "Title": "A 'robots.txt' Parser and 'Webbot'/'Spider'/'Crawler'\nPermissions Checker",
  "Version": "0.7.15.9000",
  "Authors@R": "c(\nperson(\n\"Pedro\", \"Baltazar\", role = c(\"ctb\"),\nemail = \"pedrobtz@gmail.com\"\n),\nperson(\n\"Jordan\", \"Bradford\", role = c(\"cre\"),\nemail = \"jrdnbradford@gmail.com\"\n),\nperson(\n\"Peter\", \"Meissner\", role = c(\"aut\"),\nemail = \"retep.meissner@gmail.com\"\n),\nperson(\n\"Kun\", \"Ren\", email = \"mail@renkun.me\", role = c(\"aut\", \"cph\"),\ncomment = \"Author and copyright holder of list_merge.R.\"\n),\nperson(\"Oliver\", \"Keys\", role = \"ctb\", comment = \"original release code review\"),\nperson(\"Rich\", \"Fitz John\", role = \"ctb\", comment = \"original release code review\")\n)",
  "Description": "Provides functions to download and parse 'robots.txt'\nfiles. Ultimately the package makes it easy to check if bots\n(spiders, crawler, scrapers, ...) are allowed to access\nspecific resources on a domain.",
  "License": "MIT + file LICENSE",
  "BugReports": "https://github.com/ropensci/robotstxt/issues",
  "URL": "https://docs.ropensci.org/robotstxt/,\nhttps://github.com/ropensci/robotstxt",
  "VignetteBuilder": "knitr",
  "RoxygenNote": "7.3.2",
  "Encoding": "UTF-8",
  "Config/testthat/edition": "3",
  "Config/pak/sysreqs": "libicu-dev libssl-dev",
  "Repository": "https://ropensci.r-universe.dev",
  "Date/Publication": "2024-11-15 19:32:37 UTC",
  "RemoteUrl": "https://github.com/ropensci/robotstxt",
  "RemoteRef": "HEAD",
  "RemoteSha": "d3d0a4d525e87204010d10162b01a921b05a6149",
  "NeedsCompilation": "no",
  "Packaged": {
    "Date": "2026-10-29 08:11:53 UTC",
    "User": "root"
  },
  "Author": "Pedro Baltazar [ctb],\nJordan Bradford [cre],\nPeter Meissner [aut],\nKun Ren [aut, cph] (Author and copyright holder of list_merge.R.),\nOliver Keys [ctb] (original release code review),\nRich Fitz John [ctb] (original release code review)",
  "Maintainer": "Jordan Bradford <jrdnbradford@gmail.com>",
  "_user": "ropensci",
  "_type": "src",
  "_file": "robotstxt_0.7.15.9000.tar.gz",
  "_fileid": "https://r2.ropensci.org/7d30d661a6e0dbef5b7775d1e54e45b2a62f123c06be398adab0284eedff5273",
  "_filesize": 266562,
  "_sha256": "7d30d661a6e0dbef5b7775d1e54e45b2a62f123c06be398adab0284eedff5273",
  "_expires": "2027-02-06T08:16:09.000Z",
  "_created": "2026-10-29T08:11:53.000Z",
  "_published": "2026-10-29T08:16:10.603Z",
  "_upstream": "https://github.com/ropensci/robotstxt",
  "_status": "success",
  "_maintainer": {
    "name": "Jordan Bradford",
    "email": "jrdnbradford@gmail.com",
    "login": "jrdnbradford",
    "bluesky": "@jrdnbradford.bsky.social",
    "linkedin": "in/jrdnbradford",
    "description": "Probably coding in a Dracula-themed IDE.",
    "uuid": 36420801
  },
  "_jobs": [
    {
      "job": 123335292712,
      "time": 124,
      "config": "linux-devel-x86_64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "12882098472"
    },
    {
      "job": 123335292660,
      "time": 109,
      "config": "linux-release-x86_64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "12882537332"
    },
    {
      "job": 123335292735,
      "time": 108,
      "config": "macos-oldrel-arm64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "12883015768"
    },
    {
      "job": 123335292652,
      "time": 92,
      "config": "macos-release-arm64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "12882492588"
    },
    {
      "job": 123334052933,
      "time": 131,
      "config": "pkgdown",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "12882422360"
    },
    {
      "job": 123334052988,
      "time": 190,
      "config": "source",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "12881909506"
    },
    {
      "job": 123335292627,
      "time": 91,
      "config": "wasm-release",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "12881869225"
    },
    {
      "job": 123335292657,
      "time": 83,
      "config": "windows-devel-x86_64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "12882323037"
    },
    {
      "job": 123335292755,
      "time": 75,
      "config": "windows-oldrel-x86_64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "12882337861"
    },
    {
      "job": 123335292764,
      "time": 73,
      "config": "windows-release-x86_64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "12883035280"
    }
  ],
  "_buildurl": "https://github.com/r-universe/ropensci/actions/runs/40840028371",
  "_commit": {
    "id": "d3d0a4d525e87204010d10162b01a921b05a6149",
    "author": "Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com>",
    "committer": "GitHub <noreply@github.com>",
    "message": "Merge pull request #94 from ropensci/simplify-get_robotstxt_http_get\n\nRewrite `get_robotstxt_http_get`",
    "time": 1731699157
  },
  "_distro": "resolute",
  "_host": "GitHub-Actions",
  "_registered": true,
  "_dependencies": [
    {
      "package": "R",
      "version": ">= 3.0.0",
      "role": "Depends"
    },
    {
      "package": "stringr",
      "version": ">= 1.0.0",
      "role": "Imports"
    },
    {
      "package": "httr",
      "version": ">= 1.0.0",
      "role": "Imports"
    },
    {
      "package": "spiderbar",
      "version": ">= 0.2.0",
      "role": "Imports"
    },
    {
      "package": "future.apply",
      "version": ">= 1.0.0",
      "role": "Imports"
    },
    {
      "package": "magrittr",
      "role": "Imports"
    },
    {
      "package": "utils",
      "role": "Imports"
    },
    {
      "package": "knitr",
      "role": "Suggests"
    },
    {
      "package": "rmarkdown",
      "role": "Suggests"
    },
    {
      "package": "dplyr",
      "role": "Suggests"
    },
    {
      "package": "testthat",
      "version": ">= 3.0.0",
      "role": "Suggests"
    },
    {
      "package": "covr",
      "role": "Suggests"
    },
    {
      "package": "curl",
      "role": "Suggests"
    }
  ],
  "_owner": "ropensci",
  "_selfowned": true,
  "_usedby": 4,
  "_updates": [],
  "_tags": [],
  "_topics": [
    "crawler",
    "peer-reviewed",
    "robotstxt",
    "scraper",
    "spider",
    "webscraping"
  ],
  "_stars": 69,
  "_contributors": [
    {
      "user": "petermeissner",
      "count": 248,
      "uuid": 939139
    },
    {
      "user": "jrdnbradford",
      "count": 32,
      "uuid": 36420801
    },
    {
      "user": "pedrobtz",
      "count": 19,
      "uuid": 2864346
    },
    {
      "user": "dmi3kno",
      "count": 5,
      "uuid": 13419011
    },
    {
      "user": "gittaca",
      "count": 3,
      "uuid": 17674964
    },
    {
      "user": "maelle",
      "count": 2,
      "uuid": 8360597
    },
    {
      "user": "sckott",
      "count": 2,
      "uuid": 577668
    },
    {
      "user": "karthik",
      "count": 1,
      "uuid": 138494
    },
    {
      "user": "mine-cetinkaya-rundel",
      "count": 1,
      "uuid": 5965649
    }
  ],
  "_userbio": {
    "uuid": 1200269,
    "type": "organization",
    "name": "rOpenSci",
    "followers": 1166,
    "description": "Tools and R Packages for Open Science"
  },
  "_downloads": {
    "count": 1740,
    "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/robotstxt"
  },
  "_devurl": "https://github.com/ropensci/robotstxt",
  "_pkgdown": "https://docs.ropensci.org/robotstxt/",
  "_searchresults": 465,
  "_metadata": {
    "review": {
      "id": 25,
      "status": "reviewed",
      "version": "0.1.0",
      "organization": "rOpenSci Software Review",
      "url": "https://github.com/ropensci/software-review/issues/25"
    },
    "ropensci_category": "http-tools"
  },
  "_rbuild": "4.6.1",
  "_assets": [
    "extra/citation.cff",
    "extra/citation.html",
    "extra/citation.json",
    "extra/citation.txt",
    "extra/contents.json",
    "extra/manual.pdf",
    "extra/NEWS.html",
    "extra/NEWS.txt",
    "extra/readme.html",
    "extra/readme.md",
    "extra/robotstxt.html",
    "LICENSE"
  ],
  "_homeurl": "https://github.com/ropensci/robotstxt",
  "_realowner": "ropensci",
  "_cranurl": true,
  "_releases": [
    {
      "version": "0.1.2",
      "date": "2016-02-08"
    },
    {
      "version": "0.3.2",
      "date": "2016-04-27"
    },
    {
      "version": "0.4.0",
      "date": "2017-07-16"
    },
    {
      "version": "0.4.1",
      "date": "2017-09-01"
    },
    {
      "version": "0.5.2",
      "date": "2017-11-12"
    },
    {
      "version": "0.6.0",
      "date": "2018-02-11"
    },
    {
      "version": "0.6.2",
      "date": "2018-07-18"
    },
    {
      "version": "0.7.4",
      "date": "2020-05-31"
    },
    {
      "version": "0.7.7",
      "date": "2020-06-27"
    },
    {
      "version": "0.7.8",
      "date": "2020-07-25"
    },
    {
      "version": "0.7.13",
      "date": "2020-09-03"
    },
    {
      "version": "0.7.15",
      "date": "2024-08-29"
    }
  ],
  "_exports": [
    "%>%",
    "get_robotstxt",
    "get_robotstxt_http_get",
    "get_robotstxts",
    "is_valid_robotstxt",
    "on_client_error_default",
    "on_domain_change_default",
    "on_file_type_mismatch_default",
    "on_not_found_default",
    "on_redirect_default",
    "on_server_error_default",
    "on_sub_domain_change_default",
    "on_suspect_content_default",
    "parse_robotstxt",
    "paths_allowed",
    "request_handler_handler",
    "robotstxt",
    "rt_last_http",
    "rt_request_handler"
  ],
  "_help": [
    {
      "page": "pipe",
      "title": "re-export magrittr pipe operator",
      "topics": [
        "%>%"
      ]
    },
    {
      "page": "as.list.robotstxt_text",
      "title": "Convert robotstxt_text to list",
      "topics": [
        "as.list.robotstxt_text"
      ]
    },
    {
      "page": "fix_url",
      "title": "Add http protocal if missing from URL",
      "topics": [
        "fix_url"
      ]
    },
    {
      "page": "get_robotstxt",
      "title": "Download a robots.txt file",
      "topics": [
        "get_robotstxt"
      ]
    },
    {
      "page": "get_robotstxts",
      "title": "Download multiple robotstxt files",
      "topics": [
        "get_robotstxts"
      ]
    },
    {
      "page": "guess_domain",
      "title": "Guess a domain from path",
      "topics": [
        "guess_domain"
      ]
    },
    {
      "page": "http_domain_changed",
      "title": "Check if HTTP domain changed",
      "topics": [
        "http_domain_changed"
      ]
    },
    {
      "page": "http_subdomain_changed",
      "title": "Check if HTTP subdomain changed",
      "topics": [
        "http_subdomain_changed"
      ]
    },
    {
      "page": "http_was_redirected",
      "title": "Check if HTTP redirect occurred",
      "topics": [
        "http_was_redirected"
      ]
    },
    {
      "page": "is_suspect_robotstxt",
      "title": "Check if file is valid / parsable robots.txt file",
      "topics": [
        "is_suspect_robotstxt"
      ]
    },
    {
      "page": "is_valid_robotstxt",
      "title": "Validate if a file is valid / parsable robots.txt file",
      "topics": [
        "is_valid_robotstxt"
      ]
    },
    {
      "page": "list_merge",
      "title": "Merge a number of named lists in sequential order",
      "topics": [
        "list_merge"
      ]
    },
    {
      "page": "null_to_default",
      "title": "Return default value if NULL",
      "topics": [
        "null_to_default"
      ]
    },
    {
      "page": "parse_robotstxt",
      "title": "Parse a robots.txt file",
      "topics": [
        "parse_robotstxt"
      ]
    },
    {
      "page": "paths_allowed",
      "title": "Check if a bot has permissions to access page(s)",
      "topics": [
        "paths_allowed"
      ]
    },
    {
      "page": "paths_allowed_worker_spiderbar",
      "title": "Check if a spiderbar bot has permissions to access page(s)",
      "topics": [
        "paths_allowed_worker_spiderbar"
      ]
    },
    {
      "page": "print.robotstxt",
      "title": "Print robotstxt",
      "topics": [
        "print.robotstxt"
      ]
    },
    {
      "page": "print.robotstxt_text",
      "title": "Print robotstxt's text",
      "topics": [
        "print.robotstxt_text"
      ]
    },
    {
      "page": "remove_domain",
      "title": "Remove domain from path",
      "topics": [
        "remove_domain"
      ]
    },
    {
      "page": "request_handler_handler",
      "title": "Handle robotstxt handlers",
      "topics": [
        "request_handler_handler"
      ]
    },
    {
      "page": "robotstxt",
      "title": "Generate a representation of a robots.txt file",
      "topics": [
        "robotstxt"
      ]
    },
    {
      "page": "rt_cache",
      "title": "Get the robotstxt cache",
      "topics": [
        "rt_cache"
      ]
    },
    {
      "page": "get_robotstxt_http_get",
      "title": "Storage for HTTP request response objects",
      "topics": [
        "get_robotstxt_http_get",
        "rt_last_http"
      ]
    },
    {
      "page": "rt_request_handler",
      "title": "Handle robotstxt object retrieved from HTTP request",
      "topics": [
        "on_client_error_default",
        "on_domain_change_default",
        "on_file_type_mismatch_default",
        "on_not_found_default",
        "on_redirect_default",
        "on_server_error_default",
        "on_sub_domain_change_default",
        "on_suspect_content_default",
        "rt_request_handler"
      ]
    }
  ],
  "_readme": "https://github.com/ropensci/robotstxt/raw/HEAD/README.md",
  "_rundeps": [
    "askpass",
    "cli",
    "codetools",
    "curl",
    "digest",
    "future",
    "future.apply",
    "globals",
    "glue",
    "httr",
    "jsonlite",
    "lifecycle",
    "listenv",
    "magrittr",
    "mime",
    "openssl",
    "parallelly",
    "R6",
    "Rcpp",
    "rlang",
    "spiderbar",
    "stringi",
    "stringr",
    "sys",
    "vctrs"
  ],
  "_vignettes": [
    {
      "source": "using_robotstxt.Rmd",
      "filename": "using_robotstxt.html",
      "title": "Using Robotstxt",
      "author": "Peter Meissner",
      "engine": "knitr::rmarkdown",
      "headings": [
        "Description",
        "Robots.txt files",
        "Fast food usage for the uninterested",
        "Example Usage",
        "object oriented style",
        "functional style"
      ],
      "created": "2016-01-09 20:19:32",
      "modified": "2024-08-24 11:05:33",
      "commits": 9
    }
  ],
  "_score": 9.428092529285395,
  "_indexed": true,
  "_nocasepkg": "robotstxt",
  "_universes": [
    "ropensci",
    "jrdnbradford"
  ],
  "_binaries": [
    {
      "r": "4.7.0",
      "os": "linux",
      "version": "0.7.15.9000",
      "date": "2026-10-29T08:13:59.000Z",
      "distro": "resolute",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/1e3d0fe88091796f21537c8b6b3e613d8ca16395f0653350734ecf6bd445790f",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/40840028371"
    },
    {
      "r": "4.6.1",
      "os": "linux",
      "version": "0.7.15.9000",
      "date": "2026-10-29T08:13:43.000Z",
      "distro": "resolute",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/853c65e519382bdab74c221c37b4a46522d30a0a90d98b28ce64a1a01e508436",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/40840028371"
    },
    {
      "r": "4.5.3",
      "os": "mac",
      "version": "0.7.15.9000",
      "date": "2026-10-29T08:15:21.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/3a541b9e39577e889cb8643dd2c505c60966d70423d876e79b474482af6c625a",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/40840028371"
    },
    {
      "r": "4.6.1",
      "os": "mac",
      "version": "0.7.15.9000",
      "date": "2026-10-29T08:14:55.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/682975cdd3ad3ceb0d42c1e8c7e460b1055ba9506d8b7b190cc0a751ed784860",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/40840028371"
    },
    {
      "r": "4.6.0",
      "os": "wasm",
      "version": "0.7.15.9000",
      "date": "2026-10-29T08:13:44.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/4466dd67ee029d29152c523e4299a542688e08b94c9eceae1364ec905fe9fb39",
      "status": "success",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/40840028371"
    },
    {
      "r": "4.7.0",
      "os": "win",
      "version": "0.7.15.9000",
      "date": "2026-10-29T08:13:16.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/c7a2a46b1c1af185e8fe1688f79528ec6713d6c4701c7c699a0a9c8e6714b00c",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/40840028371"
    },
    {
      "r": "4.5.3",
      "os": "win",
      "version": "0.7.15.9000",
      "date": "2026-10-29T08:13:03.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/1a1506d6aa0b8aa512991fe48a6364a230691d9be79d32f30de597f4ea107382",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/40840028371"
    },
    {
      "r": "4.6.1",
      "os": "win",
      "version": "0.7.15.9000",
      "date": "2026-10-29T08:13:02.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "https://r2.ropensci.org/8883ae15d56e4d1c6de42f44817a1833a733ba3dfb5cef6dac2b71bb2c0bea57",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/40840028371"
    }
  ]
}