Package 'elastic'

Elasticsearch alias APIs

Description

Elasticsearch alias APIs

Usage

alias_get(conn, index = NULL, alias = NULL, ignore_unavailable = FALSE, ...)

aliases_get(conn, index = NULL, alias = NULL, ignore_unavailable = FALSE, ...)

alias_exists(conn, index = NULL, alias = NULL, ...)

alias_create(
  conn,
  index,
  alias,
  filter = NULL,
  routing = NULL,
  search_routing = NULL,
  index_routing = NULL,
  ...
)

alias_rename(conn, index, alias, alias_new, ...)

alias_delete(conn, index = NULL, alias, ...)

Arguments

conn	an Elasticsearch connection object, see connect()
index	(character) An index name
alias	(character) An alias name
ignore_unavailable	(logical) What to do if an specified index name doesn't exist. If set to TRUE then those indices are ignored.
...	Curl args passed on to crul::verb-POST, crul::verb-GET, crul::verb-HEAD, or crul::verb-DELETE
filter	(named list) provides an easy way to create different "views" of the same index. Defined using Query DSL and is applied to all Search, Count, Delete By Query and More Like This operations with this alias. See examples
routing, search_routing, index_routing	(character) Associate a routing value with an alias
alias_new	(character) A new alias name, used in rename only

Details

Note that you can also create aliases when you create indices by putting the directive in the request body. See the Elasticsearch docs link

Author(s)

Scott Chamberlain [email protected]

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-aliases.html

Examples

## Not run: 
# connection setup
(x <- connect())

if (!index_exists(x, "plos")) {
  plosdat <- system.file("examples", "plos_data.json", package = "elastic")
  invisible(docs_bulk(x, plosdat))
}
if (!index_exists(x, "shakespeare")) {
  shake <- system.file("examples", "shakespeare_data_.json", package = "elastic")
  invisible(docs_bulk(x, shake))
}

# Create/update an alias
alias_create(x, index = "plos", alias = "candles")
## more than one alias
alias_create(x, index = "plos", alias = c("tables", "chairs"))

# associate an alias with two multiple different indices
alias_create(x, index = c("plos", "shakespeare"), alias = "stools")

# Retrieve a specified alias
alias_get(x, index="plos")
alias_get(x, alias="tables")
alias_get(x, alias="stools")
aliases_get(x)

# rename an alias
aliases_get(x, "plos")
alias_rename(x, index = 'plos', alias = "stools", alias_new = "plates")
aliases_get(x, "plos")

# filtered aliases
alias_create(x, index = "plos", alias = "candles", 
  filter = list(wildcard = list(title = "cell")))
## a search with the alias should give titles with cell in them
(titles <- Search(x, "candles", asdf = TRUE)$hits$hits$`_source.title`)
grepl("cell", titles, ignore.case = TRUE)

# routing
alias_create(x, index = "plos", alias = "candles", 
  routing = "1")

# Check for alias existence
alias_exists(x, index = "plos")
alias_exists(x, alias = "tables")
alias_exists(x, alias = "adsfasdf")

# Delete an alias
alias_delete(x, index = "plos", alias = "tables")
alias_exists(x, alias = "tables")

# Curl options
alias_create(x, index = "plos", alias = "tables")
aliases_get(x, alias = "tables", verbose = TRUE)

## End(Not run)

---

Use the cat Elasticsearch api.

Description

Use the cat Elasticsearch api.

Usage

cat_(conn, parse = FALSE, ...)

cat_aliases(
  conn,
  verbose = FALSE,
  index = NULL,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  expand_wildcards = "all",
  ...
)

cat_allocation(
  conn,
  verbose = FALSE,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_count(
  conn,
  verbose = FALSE,
  index = NULL,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_segments(
  conn,
  verbose = FALSE,
  index = NULL,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_health(
  conn,
  verbose = FALSE,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_indices(
  conn,
  verbose = FALSE,
  index = NULL,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_master(
  conn,
  verbose = FALSE,
  index = NULL,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_nodes(
  conn,
  verbose = FALSE,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_nodeattrs(
  conn,
  verbose = FALSE,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_pending_tasks(
  conn,
  verbose = FALSE,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_plugins(
  conn,
  verbose = FALSE,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_recovery(
  conn,
  verbose = FALSE,
  index = NULL,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_thread_pool(
  conn,
  verbose = FALSE,
  index = NULL,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_shards(
  conn,
  verbose = FALSE,
  index = NULL,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

cat_fielddata(
  conn,
  verbose = FALSE,
  index = NULL,
  fields = NULL,
  h = NULL,
  help = FALSE,
  bytes = FALSE,
  parse = FALSE,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
parse	(logical) Parse to a data.frame or not. Default: FALSE
...	Curl args passed on to crul::HttpClient
verbose	(logical) If TRUE (default) the url call used printed to console
index	(character) Index name
h	(character) Fields to return
help	(logical) Output available columns, and their meanings
bytes	(logical) Give numbers back machine friendly. Default: FALSE
expand_wildcards	(character) Whether to expand wildcard expression to concrete indices that are open, closed or both. Valid choices: 'open', 'closed', 'hidden', 'none', 'all'. default: 'all'. Available in ES >= v7.7
fields	(character) Fields to return, only used with fielddata

Details

See https://www.elastic.co/guide/en/elasticsearch/reference/current/cat.html for the cat API documentation.

Note how cat_() has an underscore at the end to avoid conflict with the function base::cat() in base R.

Examples

## Not run: 
# connection setup
(x <- connect())

# list Elasticsearch cat endpoints
cat_(x)

# Do other cat operations
cat_aliases(x)
alias_create(x, index = "plos", alias = c("tables", "chairs"))
cat_aliases(x, expand_wildcards='open')
cat_aliases(x, expand_wildcards='all')
cat_allocation(x)
cat_allocation(x, verbose=TRUE)
cat_count(x)
cat_count(x, index='plos')
cat_count(x, index='gbif')
cat_segments(x)
cat_segments(x, index='gbif')
cat_health(x)
cat_indices(x)
cat_master(x)
cat_nodes(x)
# cat_nodeattrs(x) # not available in older ES versions
cat_pending_tasks(x)
cat_plugins(x)
cat_recovery(x, verbose=TRUE)
cat_recovery(x, index='gbif')
cat_thread_pool(x)
cat_thread_pool(x, verbose=TRUE)
cat_shards(x)
cat_fielddata(x)
cat_fielddata(x, fields='body')

# capture cat data into a data.frame
cat_(x, parse = TRUE)
cat_indices(x, parse = TRUE)
cat_indices(x, parse = TRUE, verbose = TRUE)
cat_count(x, parse = TRUE)
cat_count(x, parse = TRUE, verbose = TRUE)
cat_health(x, parse = TRUE)
cat_health(x, parse = TRUE, verbose = TRUE)

# Get help - what does each column mean
head(cat_indices(x, help = TRUE, parse = TRUE))
cat_health(x, help = TRUE, parse = TRUE)
head(cat_nodes(x, help = TRUE, parse = TRUE))

# Get back only certain fields
cat_nodes(x)
cat_nodes(x, h = c('ip','port','heapPercent','name'))
cat_nodes(x, h = c('id', 'ip', 'port', 'v', 'm'))
cat_indices(x, verbose = TRUE)
cat_indices(x, verbose = TRUE, h = c('index','docs.count','store.size'))

# Get back machine friendly numbers instead of the normal human friendly
cat_indices(x, verbose = TRUE, bytes = TRUE)

# Curl options
# cat_count(x, timeout_ms = 1)

## End(Not run)

---

Elasticsearch cluster endpoints

Description

Elasticsearch cluster endpoints

Usage

cluster_settings(
  conn,
  index = NULL,
  raw = FALSE,
  callopts = list(),
  verbose = TRUE,
  ...
)

cluster_health(
  conn,
  index = NULL,
  level = NULL,
  wait_for_status = NULL,
  wait_for_relocating_shards = NULL,
  wait_for_active_shards = NULL,
  wait_for_nodes = NULL,
  timeout = NULL,
  raw = FALSE,
  callopts = list(),
  verbose = TRUE,
  ...
)

cluster_state(
  conn,
  index = NULL,
  metrics = NULL,
  raw = FALSE,
  callopts = list(),
  verbose = TRUE,
  ...
)

cluster_stats(
  conn,
  index = NULL,
  raw = FALSE,
  callopts = list(),
  verbose = TRUE,
  ...
)

cluster_reroute(conn, body, raw = FALSE, callopts = list(), ...)

cluster_pending_tasks(
  conn,
  index = NULL,
  raw = FALSE,
  callopts = list(),
  verbose = TRUE,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
index	Index
raw	If TRUE (default), data is parsed to list. If FALSE, then raw JSON.
callopts	Curl args passed on to crul::verb-POST
verbose	If TRUE (default) the url call used printed to console.
...	Further args passed on to elastic search HTTP API as parameters.
level	Can be one of cluster, indices or shards. Controls the details level of the health information returned. Defaults to cluster.
wait_for_status	One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better, i.e. green > yellow > red. By default, will not wait for any status.
wait_for_relocating_shards	A number controlling to how many relocating shards to wait for. Usually will be 0 to indicate to wait till all relocations have happened. Defaults to not wait.
wait_for_active_shards	A number controlling to how many active shards to wait for. Defaults to not wait.
wait_for_nodes	The request waits until the specified number N of nodes is available. It also accepts >=N, <=N, >N and <N. Alternatively, it is possible to use ge(N), le(N), gt(N) and lt(N) notation.
timeout	A time based parameter controlling how long to wait if one of the wait_for_XXX are provided. Defaults to 30s.
metrics	One or more of version, master_node, nodes, routing_table, metadata, and blocks. See Details.
body	Query, either a list or json.

Details

metrics param options:

• version Shows the cluster state version.

• master_node Shows the elected master_node part of the response

• nodes Shows the nodes part of the response

• routing_table Shows the routing_table part of the response. If you supply a comma separated list of indices, the returned output will only contain the indices listed.

• metadata Shows the metadata part of the response. If you supply a comma separated list of indices, the returned output will only contain the indices listed.

• blocks Shows the blocks part of the response

Additional parameters that can be passed in:

• metric A comma-separated list of metrics to display. Possible values: '_all', 'completion', 'docs', 'fielddata', 'filter_cache', 'flush', 'get', 'id_cache', 'indexing', 'merge', 'percolate', 'refresh', 'search', 'segments', 'store', 'warmer'

• completion_fields A comma-separated list of fields for completion metric (supports wildcards)

• fielddata_fields A comma-separated list of fields for fielddata metric (supports wildcards)

• fields A comma-separated list of fields for fielddata and completion metric (supports wildcards)

• groups A comma-separated list of search groups for search statistics

• allow_no_indices Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes _all string or when no indices have been specified)

• expand_wildcards Whether to expand wildcard expression to concrete indices that are open, closed or both.

• ignore_indices When performed on multiple indices, allows to ignore missing ones (default: none)

• ignore_unavailable Whether specified concrete indices should be ignored when unavailable (missing or closed)

• human Whether to return time and byte values in human-readable format.

• level Return stats aggregated at cluster, index or shard level. ('cluster', 'indices' or 'shards', default: 'indices')

• types A comma-separated list of document types for the indexing index metric

Examples

## Not run: 
# connection setup
(x <- connect())

cluster_settings(x)
cluster_health(x)

cluster_state(x)
cluster_state(x, metrics = "version")
cluster_state(x, metrics = "nodes")
cluster_state(x, metrics = c("version", "nodes"))
cluster_state(x, metrics = c("version", "nodes", 'blocks'))
cluster_state(x, "shakespeare", metrics = "metadata")
cluster_state(x, c("shakespeare", "flights"), metrics = "metadata")

cluster_stats(x)
cluster_pending_tasks(x)

body <- '{
  "commands": [ 
    {
      "move": {
        "index" : "test", "shard" : 0,
        "from_node" : "node1", "to_node" : "node2"
      }
    },
    {
      "allocate_replica" : {
        "index" : "test", "shard" : 1, "node" : "node3"
      }
    }
  ]
}'
# cluster_reroute(x, body =  body)

cluster_health(x)
# cluster_health(x, wait_for_status = "yellow", timeout = "3s")

## End(Not run)

---

Set connection details to an Elasticsearch engine.

Description

Set connection details to an Elasticsearch engine.

Usage

connect(
  host = "127.0.0.1",
  port = 9200,
  path = NULL,
  transport_schema = "http",
  user = NULL,
  pwd = NULL,
  headers = NULL,
  cainfo = NULL,
  force = FALSE,
  errors = "simple",
  warn = TRUE,
  ignore_version = FALSE,
  ...
)

Arguments

host	(character) The base host, defaults to ⁠127.0.0.1⁠
port	(character) port to connect to, defaults to 9200 (optional)
path	(character) context path that is appended to the end of the url. Default: NULL, ignored
transport_schema	(character) http or https. Default: http
user	(character) User name, if required for the connection. You can specify, but ignored for now.
pwd	(character) Password, if required for the connection. You can specify, but ignored for now.
headers	named list of headers. These headers are used in all requests
cainfo	(character) path to a crt bundle, passed to curl option cainfo
force	(logical) Force re-load of connection details. Default: FALSE
errors	(character) One of simple (Default) or complete. Simple gives http code and error message on an error, while complete gives both http code and error message, and stack trace, if available.
warn	(logical) whether to throw warnings from the Elasticsearch server when provided. Pulls warnings from response headers when given. default: TRUE. To turn these off, you can set warn=FALSE or wrap function calls in suppressWarnings(). You can also see warnings in headers by using curl verbose.
ignore_version	(logical) ignore Elasticsearch version checks? default: FALSE. Setting this to TRUE may cause some problems, it has not been fully tested yet. You may want to set this to TRUE if it's not possible to ping the root route of the Elasticsearch instance, which has the Elasticsearch version. We use the version to do alter what request is sent as different Elasticsearch versions allow different parameters.
...	additional curl options to be passed in ALL http requests

Details

The default configuration is set up for localhost access on port 9200, with no username or password.

Running this connection method doesn't ping the ES server, but only prints your connection details.

All connection details are stored within the returned object. We used to store them in various env vars, but are now contained within the object so you can have any number of connection objects and they shouldn't conflict with one another.

What is the connection object?

Creating a connection object with connect() does not create a DBI-like connection object. DBI-like objects have externalptr, etc., while connect() simply holds details about your Elasticsearch instance (host, port, authentication, etc.) that is used by other methods in this package to interact with your instances' ES API. connect() is more or less a fancy list.

You can connect to different Elasticsearch intances within the same R session by creating a separate connection object for each instance; then pass the appropriate connection object to each elastic method.

Examples

## Not run: 
# the default is set to 127.0.0.1 (i.e., localhost) and port 9200
(x <- connect())
x$make_url()
x$ping()

# pass connection object to function calls
Search(x, q = "*:*")

# set username/password (hidden in print method)
connect(user = "me", pwd = "stuff")

# set a different host
# connect(host = '162.243.152.53')
# => http://162.243.152.53:9200

# set a different port
# connect(port = 8000)
# => http://localhost:8000

# set a different context path
# connect(path = 'foo_bar')
# => http://localhost:9200/foo_bar

# set to https
# connect(transport_schema = 'https')
# => https://localhost:9200

# set headers
connect(headers = list(a = 'foobar'))

# set cainfo path (hidden in print method)
connect(cainfo = '/some/path/bundle.crt')

## End(Not run)

---

Get counts of the number of records per index.

Description

Get counts of the number of records per index.

Usage

count(conn, index = NULL, type = NULL, callopts = list(), verbose = TRUE, ...)

Arguments

conn	an Elasticsearch connection object, see connect()
index	Index, defaults to all indices
type	Document type, optional
callopts	Curl args passed on to crul::verb-GET
verbose	If TRUE (default) the url call used printed to console.
...	Further args passed on to elastic search HTTP API as parameters.

Details

See docs for the count API here https://www.elastic.co/guide/en/elasticsearch/reference/current/search-count.html

You can also get a count of documents using Search() or Search_uri() and setting size = 0

Examples

## Not run: 
# connection setup
(x <- connect())

if (!index_exists(x, "plos")) {
  plosdat <- system.file("examples", "plos_data.json",
    package = "elastic")
  plosdat <- type_remover(plosdat)
  invisible(docs_bulk(x, plosdat))
}
if (!index_exists(x, "shakespeare")) {
  shake <- system.file("examples", "shakespeare_data_.json", 
    package = "elastic")
  invisible(docs_bulk(x, shake))
}

count(x)
count(x, index='plos')
count(x, index='shakespeare')
count(x, index=c('plos','shakespeare'), q="a*")
count(x, index=c('plos','shakespeare'), q="z*")

# Curl options
count(x, callopts = list(verbose = TRUE))

## End(Not run)

---

Use the bulk API to create, index, update, or delete documents.

Description

Use the bulk API to create, index, update, or delete documents.

Usage

docs_bulk(
  conn,
  x,
  index = NULL,
  type = NULL,
  chunk_size = 1000,
  doc_ids = NULL,
  es_ids = TRUE,
  raw = FALSE,
  quiet = FALSE,
  query = list(),
  digits = NA,
  sf = NULL,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
x	A list, data.frame, or character path to a file. required.
index	(character) The index name to use. Required for data.frame input, but optional for file inputs.
type	(character) The type. default: NULL. Note that type is deprecated in Elasticsearch v7 and greater, and removed in Elasticsearch v8
chunk_size	(integer) Size of each chunk. If your data.frame is smaller thank chunk_size, this parameter is essentially ignored. We write in chunks because at some point, depending on size of each document, and Elasticsearch setup, writing a very large number of documents in one go becomes slow, so chunking can help. This parameter is ignored if you pass a file name. Default: 1000
doc_ids	An optional vector (character or numeric/integer) of document ids to use. This vector has to equal the size of the documents you are passing in, and will error if not. If you pass a factor we convert to character. Default: not passed
es_ids	(boolean) Let Elasticsearch assign document IDs as UUIDs. These are sequential, so there is order to the IDs they assign. If TRUE, doc_ids is ignored. Default: TRUE
raw	(logical) Get raw JSON back or not. If TRUE you get JSON; if FALSE you get a list. Default: FALSE
quiet	(logical) Suppress progress bar. Default: FALSE
query	(list) a named list of query parameters. optional. options include: pipeline, refresh, routing, _source, _source_excludes, _source_includes, timeout, wait_for_active_shards. See the docs bulk ES page for details
digits	digits used by the parameter of the same name by jsonlite::toJSON() to convert data to JSON before being submitted to your ES instance. default: NA
sf	used by jsonlite::toJSON() to convert sf objects. Set to "features" for conversion to GeoJSON. default: "dataframe"
...	Pass on curl options to crul::HttpClient

Details

More on the Bulk API: https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html

This function dispatches on data.frame or character input. Character input has to be a file name or the function stops with an error message.

If you pass a data.frame to this function, we by default do an index operation, that is, create the record in the index given by those parameters to the function. Down the road perhaps we will try to support other operations on the bulk API. if you pass a file, of course in that file, you can specify any operations you want.

Row names are dropped from data.frame, and top level names for a list are dropped as well.

A progress bar gives the progress for data.frames and lists - the progress bar is based around a for loop, where progress indicates progress along the iterations of the for loop, where each iteration is a chunk of data that's converted to bulk format, then pushed into Elasticsearch. The character method has no for loop, so no progress bar.

Value

A list

Document IDs

Document IDs can be passed in via the doc_ids paramater when passing in data.frame or list, but not with files. If ids are not passed to doc_ids, we assign document IDs from 1 to length of the object (rows of a data.frame, or length of a list). In the future we may allow the user to select whether they want to assign sequential numeric IDs or to allow Elasticsearch to assign IDs, which are UUIDs that are actually sequential, so you still can determine an order of your documents.

Document IDs and Factors

If you pass in ids that are of class factor, we coerce them to character with as.character. This applies to both data.frame and list inputs, but not to file inputs.

Large numbers for document IDs

Until recently, if you had very large integers for document IDs, docs_bulk failed. It should be fixed now. Let us know if not.

Missing data

As of elastic version ⁠0.7.8.9515⁠ we convert NA to null before loading into Elasticsearch. Previously, fields that had an NA were dropped - but when you read data back from Elasticsearch into R, you retain those missing values as jsonlite fills those in for you. Now, fields with NA's are made into null, and are not dropped in Elasticsearch.

Note also that null values can not be indexed or searched https://www.elastic.co/guide/en/elasticsearch/reference/5.3/null-value.html

Tips

This function returns the response from Elasticsearch, but you'll likely not be that interested in the response. If not, wrap your call to docs_bulk in invisible(), like so: invisible(docs_bulk(...))

Connections/Files

We create temporary files, and connections to those files, when data.frame's and lists are passed in to docs_bulk() (not when a file is passed in since we don't need to create a file). After inserting data into your Elasticsearch instance, we close the connections and delete the temporary files.

There are some exceptions though. When you pass in your own file, whether a tempfile or not, we don't delete those files after using them - in case you need those files again. Your own tempfile's will be cleaned up/delete when the R session ends. Non-tempfile's won't be cleaned up/deleted after the R session ends.

Elasticsearch versions that don't support type

See the type_remover() function.

See Also

Other bulk-functions: docs_bulk_create(), docs_bulk_delete(), docs_bulk_index(), docs_bulk_prep(), docs_bulk_update()

Examples

## Not run: 
# connection setup
(x <- connect())

# From a file already in newline delimited JSON format
plosdat <- system.file("examples", "plos_data.json", package = "elastic")
docs_bulk(x, plosdat)
aliases_get(x)
index_delete(x, index='plos')
aliases_get(x)

# From a data.frame
docs_bulk(x, mtcars, index = "hello")
## field names cannot contain dots
names(iris) <- gsub("\\.", "_", names(iris))
docs_bulk(x, iris, "iris")
## type can be missing, but index can not
docs_bulk(x, iris, "flowers")
## big data.frame, 53K rows, load ggplot2 package first
# res <- docs_bulk(x, diamonds, "diam")
# Search(x, "diam")$hits$total

# From a list
docs_bulk(x, apply(iris, 1, as.list), index="iris")
docs_bulk(x, apply(USArrests, 1, as.list), index="arrests")
# dim_list <- apply(diamonds, 1, as.list)
# out <- docs_bulk(x, dim_list, index="diamfromlist")

# When using in a loop
## We internally get last _id counter to know where to start on next bulk
## insert but you need to sleep in between docs_bulk calls, longer the
## bigger the data is
files <- c(system.file("examples", "test1.csv", package = "elastic"),
           system.file("examples", "test2.csv", package = "elastic"),
           system.file("examples", "test3.csv", package = "elastic"))
for (i in seq_along(files)) {
  d <- read.csv(files[[i]])
  docs_bulk(x, d, index = "testes")
  Sys.sleep(1)
}
count(x, "testes")
index_delete(x, "testes")

# You can include your own document id numbers
## Either pass in as an argument
index_create(x, "testes")
files <- c(system.file("examples", "test1.csv", package = "elastic"),
           system.file("examples", "test2.csv", package = "elastic"),
           system.file("examples", "test3.csv", package = "elastic"))
tt <- vapply(files, function(z) NROW(read.csv(z)), numeric(1))
ids <- list(1:tt[1],
           (tt[1] + 1):(tt[1] + tt[2]),
           (tt[1] + tt[2] + 1):sum(tt))
for (i in seq_along(files)) {
  d <- read.csv(files[[i]])
  docs_bulk(x, d, index = "testes", doc_ids = ids[[i]],
    es_ids = FALSE)
}
count(x, "testes")
index_delete(x, "testes")

## or include in the input data
### from data.frame's
index_create(x, "testes")
files <- c(system.file("examples", "test1_id.csv", package = "elastic"),
           system.file("examples", "test2_id.csv", package = "elastic"),
           system.file("examples", "test3_id.csv", package = "elastic"))
readLines(files[[1]])
for (i in seq_along(files)) {
  d <- read.csv(files[[i]])
  docs_bulk(x, d, index = "testes")
}
count(x, "testes")
index_delete(x, "testes")

### from lists via file inputs
index_create(x, "testes")
for (i in seq_along(files)) {
  d <- read.csv(files[[i]])
  d <- apply(d, 1, as.list)
  docs_bulk(x, d, index = "testes")
}
count(x, "testes")
index_delete(x, "testes")

# data.frame's with a single column
## this didn't use to work, but now should work
db <- paste0(sample(letters, 10), collapse = "")
index_create(x, db)
res <- data.frame(foo = 1:10)
out <- docs_bulk(x, res, index = db)
count(x, db)
index_delete(x, db)


# data.frame with a mix of actions
## make sure you use a column named 'es_action' or this won't work
## if you need to delete or update you need document IDs
if (index_exists(x, "baz")) index_delete(x, "baz")
df <- data.frame(a = 1:5, b = 6:10, c = letters[1:5], stringsAsFactors = FALSE)
invisible(docs_bulk(x, df, "baz"))
Sys.sleep(3)
(res <- Search(x, 'baz', asdf=TRUE)$hits$hits)
df[1, "a"] <- 99
df[1, "c"] <- "aa"
df[3, "c"] <- 33
df[3, "c"] <- "cc"
df$es_action <- c('update', 'delete', 'update', 'delete', 'delete')
df$id <- res$`_id`
df
invisible(docs_bulk(x, df, "baz", es_ids = FALSE))
### or es_ids = FALSE and pass in document ids to doc_ids
# invisible(docs_bulk(df, "baz", es_ids = FALSE, doc_ids = df$id))
Search(x, 'baz', asdf=TRUE)$hits$hits


# Curl options
plosdat <- system.file("examples", "plos_data.json",
  package = "elastic")
plosdat <- type_remover(plosdat)
invisible(docs_bulk(x, plosdat, verbose = TRUE))


# suppress progress bar
invisible(docs_bulk(x, mtcars, index = "hello", quiet = TRUE))
## vs.
invisible(docs_bulk(x, mtcars, index = "hello", quiet = FALSE))

## End(Not run)

---

Use the bulk API to create documents

Description

Use the bulk API to create documents

Usage

docs_bulk_create(
  conn,
  x,
  index = NULL,
  type = NULL,
  chunk_size = 1000,
  doc_ids = NULL,
  es_ids = TRUE,
  raw = FALSE,
  quiet = FALSE,
  query = list(),
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
x	A list, data.frame, or character path to a file. required.
index	(character) The index name to use. Required for data.frame input, but optional for file inputs.
type	(character) The type. default: NULL. Note that type is deprecated in Elasticsearch v7 and greater, and removed in Elasticsearch v8
chunk_size	(integer) Size of each chunk. If your data.frame is smaller thank chunk_size, this parameter is essentially ignored. We write in chunks because at some point, depending on size of each document, and Elasticsearch setup, writing a very large number of documents in one go becomes slow, so chunking can help. This parameter is ignored if you pass a file name. Default: 1000
doc_ids	An optional vector (character or numeric/integer) of document ids to use. This vector has to equal the size of the documents you are passing in, and will error if not. If you pass a factor we convert to character. Default: not passed
es_ids	(boolean) Let Elasticsearch assign document IDs as UUIDs. These are sequential, so there is order to the IDs they assign. If TRUE, doc_ids is ignored. Default: TRUE
raw	(logical) Get raw JSON back or not. If TRUE you get JSON; if FALSE you get a list. Default: FALSE
quiet	(logical) Suppress progress bar. Default: FALSE
query	(list) a named list of query parameters. optional. options include: pipeline, refresh, routing, _source, _source_excludes, _source_includes, timeout, wait_for_active_shards. See the docs bulk ES page for details
...	Pass on curl options to crul::HttpClient

Details

For doing create with a file already prepared for the bulk API, see docs_bulk()

Only data.frame's are supported for now.

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html

See Also

Other bulk-functions: docs_bulk_delete(), docs_bulk_index(), docs_bulk_prep(), docs_bulk_update(), docs_bulk()

Examples

## Not run: 
x <- connect()
if (index_exists(x, "foobar")) index_delete(x, "foobar")

df <- data.frame(name = letters[1:3], size = 1:3, id = 100:102)
docs_bulk_create(x, df, 'foobar', es_ids = FALSE)
Search(x, "foobar", asdf = TRUE)$hits$hits

# more examples
docs_bulk_create(x, mtcars, index = "hello")
## field names cannot contain dots
names(iris) <- gsub("\\.", "_", names(iris))
docs_bulk_create(x, iris, "iris")
## type can be missing, but index can not
docs_bulk_create(x, iris, "flowers")
## big data.frame, 53K rows, load ggplot2 package first
# res <- docs_bulk_create(x, diamonds, "diam")
# Search(x, "diam")$hits$total$value

## End(Not run)

---

Use the bulk API to delete documents

Description

Use the bulk API to delete documents

Usage

docs_bulk_delete(
  conn,
  x,
  index = NULL,
  type = NULL,
  chunk_size = 1000,
  doc_ids = NULL,
  raw = FALSE,
  quiet = FALSE,
  query = list(),
  digits = NA,
  sf = NULL,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
x	A list, data.frame, or character path to a file. required.
index	(character) The index name to use. Required for data.frame input, but optional for file inputs.
type	(character) The type. default: NULL. Note that type is deprecated in Elasticsearch v7 and greater, and removed in Elasticsearch v8
chunk_size	(integer) Size of each chunk. If your data.frame is smaller thank chunk_size, this parameter is essentially ignored. We write in chunks because at some point, depending on size of each document, and Elasticsearch setup, writing a very large number of documents in one go becomes slow, so chunking can help. This parameter is ignored if you pass a file name. Default: 1000
doc_ids	An optional vector (character or numeric/integer) of document ids to use. This vector has to equal the size of the documents you are passing in, and will error if not. If you pass a factor we convert to character. Default: not passed
raw	(logical) Get raw JSON back or not. If TRUE you get JSON; if FALSE you get a list. Default: FALSE
quiet	(logical) Suppress progress bar. Default: FALSE
query	(list) a named list of query parameters. optional. options include: pipeline, refresh, routing, _source, _source_excludes, _source_includes, timeout, wait_for_active_shards. See the docs bulk ES page for details
digits, sf	ignored, used in other docs bulk functions but not used here
...	Pass on curl options to crul::HttpClient

Details

For doing deletes with a file already prepared for the bulk API, see docs_bulk()

Only data.frame's are supported for now.

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html

See Also

Other bulk-functions: docs_bulk_create(), docs_bulk_index(), docs_bulk_prep(), docs_bulk_update(), docs_bulk()

Examples

## Not run: 
x <- connect()
if (index_exists(x, "foobar")) index_delete(x, "foobar")

df <- data.frame(name = letters[1:3], size = 1:3, id = 100:102)
invisible(docs_bulk(x, df, 'foobar', es_ids = FALSE))
Search(x, "foobar", asdf = TRUE)$hits$hits

# delete using doc ids from the data.frame you used to create
invisible(docs_bulk_delete(x, df, index = 'foobar'))
Search(x, "foobar", asdf = TRUE)$hits$total$value

# delete by passing in doc ids
## recreate data first
if (index_exists(x, "foobar")) index_delete(x, "foobar")
df <- data.frame(name = letters[1:3], size = 1:3, id = 100:102)
invisible(docs_bulk(x, df, 'foobar', es_ids = FALSE))
docs_bulk_delete(x, df, index = 'foobar', doc_ids = df$id)
Search(x, "foobar", asdf = TRUE)$hits$total$value

## End(Not run)

---

Use the bulk API to index documents

Description

Use the bulk API to index documents

Usage

docs_bulk_index(
  conn,
  x,
  index = NULL,
  type = NULL,
  chunk_size = 1000,
  doc_ids = NULL,
  es_ids = TRUE,
  raw = FALSE,
  quiet = FALSE,
  query = list(),
  digits = NA,
  sf = NULL,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
x	A list, data.frame, or character path to a file. required.
index	(character) The index name to use. Required for data.frame input, but optional for file inputs.
type	(character) The type. default: NULL. Note that type is deprecated in Elasticsearch v7 and greater, and removed in Elasticsearch v8
chunk_size	(integer) Size of each chunk. If your data.frame is smaller thank chunk_size, this parameter is essentially ignored. We write in chunks because at some point, depending on size of each document, and Elasticsearch setup, writing a very large number of documents in one go becomes slow, so chunking can help. This parameter is ignored if you pass a file name. Default: 1000
doc_ids	An optional vector (character or numeric/integer) of document ids to use. This vector has to equal the size of the documents you are passing in, and will error if not. If you pass a factor we convert to character. Default: not passed
es_ids	(boolean) Let Elasticsearch assign document IDs as UUIDs. These are sequential, so there is order to the IDs they assign. If TRUE, doc_ids is ignored. Default: TRUE
raw	(logical) Get raw JSON back or not. If TRUE you get JSON; if FALSE you get a list. Default: FALSE
quiet	(logical) Suppress progress bar. Default: FALSE
query	(list) a named list of query parameters. optional. options include: pipeline, refresh, routing, _source, _source_excludes, _source_includes, timeout, wait_for_active_shards. See the docs bulk ES page for details
digits	digits used by the parameter of the same name by jsonlite::toJSON() to convert data to JSON before being submitted to your ES instance. default: NA
sf	used by jsonlite::toJSON() to convert sf objects. Set to "features" for conversion to GeoJSON. default: "dataframe"
...	Pass on curl options to crul::HttpClient

Details

For doing index with a file already prepared for the bulk API, see docs_bulk()

Only data.frame's are supported for now.

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html

See Also

Other bulk-functions: docs_bulk_create(), docs_bulk_delete(), docs_bulk_prep(), docs_bulk_update(), docs_bulk()

Examples

## Not run: 
x <- connect()
if (index_exists(x, "foobar")) index_delete(x, "foobar")

df <- data.frame(name = letters[1:3], size = 1:3, id = 100:102)
docs_bulk_index(x, df, 'foobar')
docs_bulk_index(x, df, 'foobar', es_ids = FALSE)
Search(x, "foobar", asdf = TRUE)$hits$hits

# more examples
docs_bulk_index(x, mtcars, index = "hello")
## field names cannot contain dots
names(iris) <- gsub("\\.", "_", names(iris))
docs_bulk_index(x, iris, "iris")
## type can be missing, but index can not
docs_bulk_index(x, iris, "flowers")
## big data.frame, 53K rows, load ggplot2 package first
# res <- docs_bulk_index(x, diamonds, "diam")
# Search(x, "diam")$hits$total$value

## End(Not run)

---

Use the bulk API to prepare bulk format data

Description

Use the bulk API to prepare bulk format data

Usage

docs_bulk_prep(
  x,
  index,
  path,
  type = NULL,
  chunk_size = 1000,
  doc_ids = NULL,
  quiet = FALSE,
  digits = NA,
  sf = NULL
)

Arguments

x	A data.frame or a list. required.
index	(character) The index name. required.
path	(character) Path to the file. If data is broken into chunks, we'll use this path as the prefix, and suffix each file path with a number. required.
type	(character) The type. default: NULL. Note that type is deprecated in Elasticsearch v7 and greater, and removed in Elasticsearch v8
chunk_size	(integer) Size of each chunk. If your data.frame is smaller thank chunk_size, this parameter is essentially ignored. We write in chunks because at some point, depending on size of each document, and Elasticsearch setup, writing a very large number of documents in one go becomes slow, so chunking can help. This parameter is ignored if you pass a file name. Default: 1000
doc_ids	An optional vector (character or numeric/integer) of document ids to use. This vector has to equal the size of the documents you are passing in, and will error if not. If you pass a factor we convert to character. Default: not passed
quiet	(logical) Suppress progress bar. Default: FALSE
digits	digits used by the parameter of the same name by jsonlite::toJSON() to convert data to JSON before being submitted to your ES instance. default: NA
sf	used by jsonlite::toJSON() to convert sf objects. Set to "features" for conversion to GeoJSON. default: "dataframe"

Value

File path(s). By default we use temporary files; these are cleaned up at the end of a session

Tempfiles

In docs_bulk we create temporary files in some cases, and delete those before the function exits. However, we don't clean up those files in this function because the point of the function is to create the newline delimited JSON files that you need. Tempfiles are cleaned up when you R session ends though - be aware of that. If you want to keep the files make sure to move them outside of the temp directory.

See Also

Other bulk-functions: docs_bulk_create(), docs_bulk_delete(), docs_bulk_index(), docs_bulk_update(), docs_bulk()

Examples

## Not run: 
# From a data.frame
ff <- tempfile(fileext = ".json")
docs_bulk_prep(mtcars, index = "hello", path = ff)
readLines(ff)

## field names cannot contain dots
names(iris) <- gsub("\\.", "_", names(iris))
docs_bulk_prep(iris, "iris", path = tempfile(fileext = ".json"))

## type can be missing, but index can not
docs_bulk_prep(iris, "flowers", path = tempfile(fileext = ".json"))

# From a list
docs_bulk_prep(apply(iris, 1, as.list), index="iris",
   path = tempfile(fileext = ".json"))
docs_bulk_prep(apply(USArrests, 1, as.list), index="arrests",
   path = tempfile(fileext = ".json"))

# when chunking
## multiple files created, one for each chunk
bigiris <- do.call("rbind", replicate(30, iris, FALSE))
docs_bulk_prep(bigiris, index = "big", path = tempfile(fileext = ".json"))

# When using in a loop
## We internally get last _id counter to know where to start on next bulk
## insert but you need to sleep in between docs_bulk_prep calls, longer the
## bigger the data is
files <- c(system.file("examples", "test1.csv", package = "elastic"),
           system.file("examples", "test2.csv", package = "elastic"),
           system.file("examples", "test3.csv", package = "elastic"))
paths <- vector("list", length = length(files))
for (i in seq_along(files)) {
  d <- read.csv(files[[i]])
  paths[i] <- docs_bulk_prep(d, index = "stuff",
     path = tempfile(fileext = ".json"))
}
unlist(paths)

# You can include your own document id numbers
## Either pass in as an argument
files <- c(system.file("examples", "test1.csv", package = "elastic"),
           system.file("examples", "test2.csv", package = "elastic"),
           system.file("examples", "test3.csv", package = "elastic"))
tt <- vapply(files, function(z) NROW(read.csv(z)), numeric(1))
ids <- list(1:tt[1],
           (tt[1] + 1):(tt[1] + tt[2]),
           (tt[1] + tt[2] + 1):sum(tt))
paths <- vector("list", length = length(files))
for (i in seq_along(files)) {
  d <- read.csv(files[[i]])
  paths[i] <- docs_bulk_prep(d, index = "testes",
    doc_ids = ids[[i]], path = tempfile(fileext = ".json"))
}
unlist(paths)

## or include in the input data
### from data.frame's
files <- c(system.file("examples", "test1_id.csv", package = "elastic"),
           system.file("examples", "test2_id.csv", package = "elastic"),
           system.file("examples", "test3_id.csv", package = "elastic"))
paths <- vector("list", length = length(files))
for (i in seq_along(files)) {
  d <- read.csv(files[[i]])
  paths[i] <- docs_bulk_prep(d, index = "testes",
     path = tempfile(fileext = ".json"))
}
unlist(paths)

### from lists via file inputs
paths <- vector("list", length = length(files))
for (i in seq_along(files)) {
  d <- read.csv(files[[i]])
  d <- apply(d, 1, as.list)
  paths[i] <- docs_bulk_prep(d, index = "testes",
      path = tempfile(fileext = ".json"))
}
unlist(paths)


# A mix of actions
## make sure you use a column named 'es_action' or this won't work
## if you need to delete or update you need document IDs
if (index_exists(x, "baz")) index_delete(x, "baz")
df <- data.frame(a = 1:5, b = 6:10, c = letters[1:5], stringsAsFactors = FALSE)
f <- tempfile(fileext = ".json")
invisible(docs_bulk_prep(df, "baz", f))
cat(readLines(f), sep = "\n")
docs_bulk(x, f)
Sys.sleep(2)
(res <- Search(x, 'baz', asdf=TRUE)$hits$hits)

df[1, "a"] <- 99
df[1, "c"] <- "aa"
df[3, "c"] <- 33
df[3, "c"] <- "cc"
df$es_action <- c('update', 'delete', 'update', 'delete', 'delete')
df$id <- res$`_id`
df
f <- tempfile(fileext = ".json")
invisible(docs_bulk_prep(df, "baz", path = f, doc_ids = df$id))
cat(readLines(f), sep = "\n")
docs_bulk(x, f)


# suppress progress bar
docs_bulk_prep(mtcars, index = "hello",
  path = tempfile(fileext = ".json"), quiet = TRUE)
## vs.
docs_bulk_prep(mtcars, index = "hello",
  path = tempfile(fileext = ".json"), quiet = FALSE)

## End(Not run)

---

Use the bulk API to update documents

Description

Use the bulk API to update documents

Usage

docs_bulk_update(
  conn,
  x,
  index = NULL,
  type = NULL,
  chunk_size = 1000,
  doc_ids = NULL,
  raw = FALSE,
  quiet = FALSE,
  query = list(),
  digits = NA,
  sf = NULL,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
x	A list, data.frame, or character path to a file. required.
index	(character) The index name to use. Required for data.frame input, but optional for file inputs.
type	(character) The type. default: NULL. Note that type is deprecated in Elasticsearch v7 and greater, and removed in Elasticsearch v8
chunk_size	(integer) Size of each chunk. If your data.frame is smaller thank chunk_size, this parameter is essentially ignored. We write in chunks because at some point, depending on size of each document, and Elasticsearch setup, writing a very large number of documents in one go becomes slow, so chunking can help. This parameter is ignored if you pass a file name. Default: 1000
doc_ids	An optional vector (character or numeric/integer) of document ids to use. This vector has to equal the size of the documents you are passing in, and will error if not. If you pass a factor we convert to character. Default: not passed
raw	(logical) Get raw JSON back or not. If TRUE you get JSON; if FALSE you get a list. Default: FALSE
quiet	(logical) Suppress progress bar. Default: FALSE
query	(list) a named list of query parameters. optional. options include: pipeline, refresh, routing, _source, _source_excludes, _source_includes, timeout, wait_for_active_shards. See the docs bulk ES page for details
digits	digits used by the parameter of the same name by jsonlite::toJSON() to convert data to JSON before being submitted to your ES instance. default: NA
sf	used by jsonlite::toJSON() to convert sf objects. Set to "features" for conversion to GeoJSON. default: "dataframe"
...	Pass on curl options to crul::HttpClient

Details

• doc_as_upsert - is set to TRUE for all records

For doing updates with a file already prepared for the bulk API, see docs_bulk()

Only data.frame's are supported for now.

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html#bulk-update

See Also

Other bulk-functions: docs_bulk_create(), docs_bulk_delete(), docs_bulk_index(), docs_bulk_prep(), docs_bulk()

Examples

## Not run: 
x <- connect()
if (index_exists(x, "foobar")) index_delete(x, "foobar")

df <- data.frame(name = letters[1:3], size = 1:3, id = 100:102)
invisible(docs_bulk(x, df, 'foobar', es_ids = FALSE))

# add new rows in existing fields
(df2 <- data.frame(size = c(45, 56), id = 100:101))
(df2 <- data.frame(size = c(45, 56)))
df2$`_id` <- 100:101
df2
Search(x, "foobar", asdf = TRUE)$hits$hits
invisible(docs_bulk_update(x, df2, index = 'foobar'))
Search(x, "foobar", asdf = TRUE)$hits$hits

# add new fields (and new rows by extension)
(df3 <- data.frame(color = c("blue", "red", "green"), id = 100:102))
Search(x, "foobar", asdf = TRUE)$hits$hits
invisible(docs_bulk_update(x, df3, index = 'foobar'))
Sys.sleep(2) # wait for a few sec to make sure you see changes reflected
Search(x, "foobar", asdf = TRUE)$hits$hits

## End(Not run)

---

Create a document

Description

Create a document

Usage

docs_create(
  conn,
  index,
  body,
  type = NULL,
  id = NULL,
  version = NULL,
  version_type = NULL,
  op_type = NULL,
  routing = NULL,
  parent = NULL,
  timestamp = NULL,
  ttl = NULL,
  refresh = NULL,
  timeout = NULL,
  callopts = list(),
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
index	(character) The name of the index. Required
body	The document
type	(character) The type of the document. optional
id	(numeric/character) The document ID. Can be numeric or character. Optional. if not provided, Elasticsearch creates the ID for you as a UUID.
version	(character) Explicit version number for concurrency control
version_type	(character) Specific version type. One of internal, external, external_gte, or force
op_type	(character) Operation type. One of create, or ...
routing	(character) Specific routing value
parent	(numeric) A parent document ID
timestamp	(date) Explicit timestamp for the document
ttl	(aka “time to live”) Expiration time for the document. Expired documents will be expunged automatically. The expiration date that will be set for a document with a provided ttl is relative to the timestamp of the document, meaning it can be based on the time of indexing or on any time provided. The provided ttl must be strictly positive and can be a number (in milliseconds) or any valid time value (e.g, 86400000, 1d).
refresh	(logical) Refresh the index after performing the operation
timeout	(character) Explicit operation timeout, e.g,. 5m (for 5 minutes)
callopts	Curl options passed on to crul::HttpClient
...	Further args to query DSL

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html

Examples

## Not run: 
(x <- connect())

if (!index_exists(x, 'plos')) {
  plosdat <- system.file("examples", "plos_data.json",
    package = "elastic")
  plosdat <- type_remover(plosdat)
  invisible(docs_bulk(x, plosdat))
}

# give a document id
z <- docs_create(x, index = 'plos', id = 1002,
  body = list(id = "12345", title = "New title"))
z
# and the document is there now
docs_get(x, index = 'plos', id = 1002)

# let Elasticsearch create the document id for you
z <- docs_create(x, index='plos', body=list(id="6789", title="Some title"))
z
# and the document is there now
docs_get(x, index='plos', id=z$`_id`)

## End(Not run)

---

Delete a document

Description

Delete a document

Usage

docs_delete(
  conn,
  index,
  id,
  type = NULL,
  refresh = NULL,
  routing = NULL,
  timeout = NULL,
  version = NULL,
  version_type = NULL,
  callopts = list(),
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
index	(character) The name of the index. Required
id	(numeric/character) The document ID. Can be numeric or character. Required
type	(character) The type of the document. optional
refresh	(logical) Refresh the index after performing the operation
routing	(character) Specific routing value
timeout	(character) Explicit operation timeout, e.g,. 5m (for 5 minutes)
version	(character) Explicit version number for concurrency control
version_type	(character) Specific version type. One of internal or external
callopts	Curl args passed on to crul::HttpClient
...	Further args to query DSL

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete.html

Examples

## Not run: 
(x <- connect())
x$ping()

if (!index_exists(x, "plos")) {
 plosdat <- system.file("examples", "plos_data.json",
    package = "elastic")
 plosdat <- type_remover(plosdat)
 docs_bulk(x, plosdat)
}

# delete a document
if (!docs_get(x, index='plos', id=36, exists=TRUE)) {
  docs_create(x, index='plos', id=36, 
    body = list(id="12345", title="New title")
  )
}
docs_get(x, index='plos', id=36)
docs_delete(x, index='plos', id=36)
# docs_get(x, index='plos', id=36) # and the document is gone

## End(Not run)

---

Delete documents by query

Description

delete documents by query via a POST request

Usage

docs_delete_by_query(
  conn,
  index,
  body,
  type = NULL,
  conflicts = NULL,
  routing = NULL,
  scroll_size = NULL,
  refresh = NULL,
  wait_for_completion = NULL,
  wait_for_active_shards = NULL,
  timeout = NULL,
  scroll = NULL,
  requests_per_second = NULL,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
index	(character) The name of the index. Required
body	(character/json) query to be passed on to POST request body
type	(character) The type of the document. optional
conflicts	(character) If you’d like to count version conflicts rather than cause them to abort then set conflicts=proceed
routing	(character) Specific routing value
scroll_size	(integer) By default uses scroll batches of 1000. Change batch size with this parameter.
refresh	(logical) Refresh the index after performing the operation
wait_for_completion	(logical) If wait_for_completion=FALSE then Elasticsearch will perform some preflight checks, launch the request, and then return a task which can be used with Tasks APIs to cancel or get the status of the task. Elasticsearch will also create a record of this task as a document at .tasks/task/$taskId. This is yours to keep or remove as you see fit. When you are done with it, delete it so Elasticsearch can reclaim the space it uses. Default: TRUE
wait_for_active_shards	(logical) controls how many copies of a shard must be active before proceeding with the request.
timeout	(character) Explicit operation timeout, e.g,. 5m (for 5 minutes)
scroll	(integer) control how long the "search context" is kept alive, eg scroll='10m', by default it’s 5 minutes (⁠5m⁠)
requests_per_second	(integer) any positive decimal number (1.4, 6, 1000, etc); throttles rate at which ⁠_delete_by_query⁠ issues batches of delete operations by padding each batch with a wait time. The throttling can be disabled by setting requests_per_second=-1
...	Curl args passed on to crul::verb-POST

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html

See Also

docs_update_by_query()

Examples

## Not run: 
(x <- connect())
x$ping()

plosdat <- system.file("examples", "plos_data.json",
  package = "elastic")
plosdat <- type_remover(plosdat)
if (!index_exists(x, "plos")) invisible(docs_bulk(x, plosdat))

# delete with fuzzy matching
body <- '{
  "query": { 
    "match": {
      "title": {
        "query": "cells",
        "fuzziness": 1
      }
    }
  }
}'
docs_delete_by_query(x, index='plos', body = body) 

# delete with no fuzziness
if (index_exists(x, "plos")) index_delete(x, 'plos')
invisible(docs_bulk(x, plosdat))
count(x, "plos")
body <- '{
  "query": { 
    "match": {
      "title": {
        "query": "cells",
        "fuzziness": 0
      }
    }
  }
}'
docs_delete_by_query(x, index='plos', body = body)

# delete all docs with match_all query
if (index_exists(x, "plos")) index_delete(x, 'plos')
invisible(docs_bulk(x, plosdat))
body <- '{
  "query": { 
    "match_all": {}
  }
}'
docs_delete_by_query(x, index='plos', body = body)

# put plos back in 
if (index_exists(x, "plos")) index_delete(x, 'plos')
invisible(docs_bulk(x, plosdat))

# delete docs from more than one index
foo <- system.file("examples/foo.json", package = "elastic")
if (!index_exists(x, "foo")) invisible(docs_bulk(x, foo))
bar <- system.file("examples/bar.json", package = "elastic")
if (!index_exists(x, "bar")) invisible(docs_bulk(x, bar))

body <- '{
  "query": { 
    "match_all": {}
  }
}'
docs_delete_by_query(x, index=c('foo','bar'), 
  body = body, verbose = TRUE)

## End(Not run)

---

Get documents

Description

Get documents

Usage

docs_get(
  conn,
  index,
  id,
  type = NULL,
  source = NULL,
  fields = NULL,
  source_includes = NULL,
  source_excludes = NULL,
  exists = FALSE,
  raw = FALSE,
  callopts = list(),
  verbose = TRUE,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
index	(character) The name of the index. Required
id	(numeric/character) The document ID. Can be numeric or character. Required
type	(character) The type of the document. optional
source	(logical) If TRUE (default), return source. note that it is actually set to NULL in the function definition, but within Elasticsearch, it returns the source by default. alternatively, you can pass a vector of field names to return.
fields	Fields to return from the response object.
source_includes, source_excludes	(character) fields to include in the returned document, or to exclude. a character vector
exists	(logical) Only return a logical as to whether the document exists or not.
raw	If TRUE (default), data is parsed to list. If FALSE, then raw JSON.
callopts	Curl args passed on to crul::HttpClient
verbose	If TRUE (default) the url call used printed to console.
...	Further args passed on to elastic search HTTP API as parameters.

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html

Examples

## Not run: 
(x <- connect())

if (!index_exists(x, "shakespeare")) {
  shakespeare <- system.file("examples", "shakespeare_data_.json",
    package = "elastic")
  shakespeare <- type_remover(shakespeare)
  invisible(docs_bulk(x, shakespeare))
}

docs_get(x, index='shakespeare', id=10)
docs_get(x, index='shakespeare', id=12)
docs_get(x, index='shakespeare', id=12, source=TRUE)

# Get certain fields
if (gsub("\\.", "", x$ping()$version$number) < 500) {
  ### ES < v5
  docs_get(x, index='shakespeare', id=10, fields='play_name')
  docs_get(x, index='shakespeare', id=10, fields=c('play_name','speaker'))
} else {
  ### ES > v5
  docs_get(x, index='shakespeare', id=10, source='play_name')
  docs_get(x, index='shakespeare', id=10, source=c('play_name','speaker'))
}

# Just test for existence of the document
docs_get(x, index='plos', id=1, exists=TRUE)
docs_get(x, index='plos', id=123456, exists=TRUE)

# source includes / excludes
docs_get(x, index='shakespeare', id=10, source_includes = "play_name")
docs_get(x, index='shakespeare', id=10, source_excludes = "play_name")

## End(Not run)

---

Get multiple documents via the multiple get API

Description

Get multiple documents via the multiple get API

Usage

docs_mget(
  conn,
  index = NULL,
  type = NULL,
  ids = NULL,
  type_id = NULL,
  index_type_id = NULL,
  source = NULL,
  fields = NULL,
  raw = FALSE,
  callopts = list(),
  verbose = TRUE,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
index	Index. Required.
type	Document type. Required.
ids	More than one document id, see examples.
type_id	List of vectors of length 2, each with an element for type and id.
index_type_id	List of vectors of length 3, each with an element for index, type, and id.
source	(logical) If TRUE, return source.
fields	Fields to return from the response object.
raw	If TRUE (default), data is parsed to list. If FALSE, then raw JSON.
callopts	Curl args passed on to HttpClient
verbose	If TRUE (default) the url call used printed to console.
...	Further args passed on to elastic search HTTP API as parameters.

Details

You can pass in one of three combinations of parameters:

• Pass in something for index, type, and id. This is the simplest, allowing retrieval from the same index, same type, and many ids.

• Pass in only index and type_id - this allows you to get multiple documents from the same index, but from different types.

• Pass in only index_type_id - this is so that you can get multiple documents from different indexes and different types.

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html

Examples

## Not run: 
(x <- connect())

if (!index_exists(x, 'plos')) {
  plosdat <- system.file("examples", "plos_data.json",
    package = "elastic")
  plosdat <- type_remover(plosdat)
  invisible(docs_bulk(x, plosdat))
}

# same index, many ids
docs_mget(x, index="plos", ids=c(9,10))

# Same index and type
docs_mget(x, index="plos", type="_doc", ids=c(9,10))

tmp <- docs_mget(x, index="plos", ids=c(9, 10), raw=TRUE)
es_parse(tmp)
docs_mget(x, index="plos", ids=c(9, 10), source='title')
docs_mget(x, index="plos", ids=c(14, 19), source=TRUE)

# curl options
docs_mget(x, index="plos", ids=1:2, callopts=list(verbose=TRUE))

# Same index, but different types
if (index_exists(x, 'shakespeare')) index_delete(x, 'shakespeare')
shakedat <- system.file("examples", "shakespeare_data.json",
  package = "elastic")
invisible(docs_bulk(x, shakedat))

docs_mget(x, index="shakespeare", type_id=list(c("scene",1), c("line",20)))
docs_mget(x, index="shakespeare", type_id=list(c("scene",1), c("line",20)),
  source='play_name')

# Different indices and different types pass in separately
docs_mget(x, index_type_id = list(
  c("shakespeare", "line", 20),
  c("plos", "article", 1)
 )
)

## End(Not run)

---

Update a document

Description

Update a document

Usage

docs_update(
  conn,
  index,
  id,
  body,
  type = NULL,
  fields = NULL,
  source = NULL,
  version = NULL,
  version_type = NULL,
  routing = NULL,
  parent = NULL,
  timestamp = NULL,
  ttl = NULL,
  refresh = NULL,
  timeout = NULL,
  retry_on_conflict = NULL,
  wait_for_active_shards = NULL,
  detect_noop = NULL,
  callopts = list(),
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
index	(character) The name of the index. Required
id	(numeric/character) The document ID. Can be numeric or character. Required
body	The document, either a list or json
type	(character) The type of the document. optional
fields	A comma-separated list of fields to return in the response
source	Allows to control if and how the updated source should be returned in the response. By default the updated source is not returned.
version	(character) Explicit version number for concurrency control
version_type	(character) Specific version type. One of internal, external, external_gte, or force
routing	(character) Specific routing value
parent	ID of the parent document. Is is only used for routing and when for the upsert request
timestamp	(date) Explicit timestamp for the document
ttl	(aka “time to live”) Expiration time for the document. Expired documents will be expunged automatically. The expiration date that will be set for a document with a provided ttl is relative to the timestamp of the document, meaning it can be based on the time of indexing or on any time provided. The provided ttl must be strictly positive and can be a number (in milliseconds) or any valid time value (e.g, 86400000, 1d).
refresh	Refresh the index after performing the operation.
timeout	(character) Explicit operation timeout, e.g,. 5m (for 5 minutes)
retry_on_conflict	Specify how many times should the operation be retried when a conflict occurs (default: 0)
wait_for_active_shards	The number of shard copies required to be active before proceeding with the update operation.
detect_noop	(logical) Specifying TRUE will cause Elasticsearch to check if there are changes and, if there aren't, turn the update request into a noop.
callopts	Curl options passed on to crul::HttpClient
...	Further args to query DSL

Examples

## Not run: 
(x <- connect())
if (!index_exists(x, 'plos')) {
  plosdat <- system.file("examples", "plos_data.json",
    package = "elastic")
  plosdat <- type_remover(plosdat)
  invisible(docs_bulk(x, plosdat))
}

docs_create(x, index='plos', id=1002,
  body=list(id="12345", title="New title"))
# and the document is there now
docs_get(x, index='plos', id=1002)
# update the document
docs_update(x, index='plos', id=1002,
  body = list(doc = list(title = "Even newer title again")))
# get it again, notice changes
docs_get(x, index='plos', id=1002)

if (!index_exists(x, 'stuffthings')) {
  index_create(x, "stuffthings")
}
docs_create(x, index='stuffthings', id=1,
  body=list(name = "foo", what = "bar"))
docs_update(x, index='stuffthings', id=1,
  body = list(doc = list(name = "hello", what = "bar")),
  source = 'name')

## End(Not run)

---

Update documents by query

Description

update documents by query via a POST request

Usage

docs_update_by_query(
  conn,
  index,
  body = NULL,
  type = NULL,
  conflicts = NULL,
  routing = NULL,
  scroll_size = NULL,
  refresh = NULL,
  wait_for_completion = NULL,
  wait_for_active_shards = NULL,
  timeout = NULL,
  scroll = NULL,
  requests_per_second = NULL,
  pipeline = NULL,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
index	(character) The name of the index. Required
body	(character/json) query to be passed on to POST request body
type	(character) The type of the document. optional
conflicts	(character) If you’d like to count version conflicts rather than cause them to abort then set conflicts=proceed
routing	(character) Specific routing value
scroll_size	(integer) By default uses scroll batches of 1000. Change batch size with this parameter.
refresh	(logical) Refresh the index after performing the operation
wait_for_completion	(logical) If wait_for_completion=FALSE then Elasticsearch will perform some preflight checks, launch the request, and then return a task which can be used with Tasks APIs to cancel or get the status of the task. Elasticsearch will also create a record of this task as a document at .tasks/task/$taskId. This is yours to keep or remove as you see fit. When you are done with it, delete it so Elasticsearch can reclaim the space it uses. Default: TRUE
wait_for_active_shards	(logical) controls how many copies of a shard must be active before proceeding with the request.
timeout	(character) Explicit operation timeout, e.g,. 5m (for 5 minutes)
scroll	(integer) control how long the "search context" is kept alive, eg scroll='10m', by default it’s 5 minutes (⁠5m⁠)
requests_per_second	(integer) any positive decimal number (1.4, 6, 1000, etc); throttles rate at which ⁠_delete_by_query⁠ issues batches of delete operations by padding each batch with a wait time. The throttling can be disabled by setting requests_per_second=-1
pipeline	(character) a pipeline name
...	Curl args passed on to crul::verb-POST

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html https://www.elastic.co/guide/en/elasticsearch/painless/current/painless-api-reference.html

See Also

docs_delete_by_query()

Examples

## Not run: 
(x <- connect())
x$ping()

omdb <- system.file("examples", "omdb.json", package = "elastic")
omdb <- type_remover(omdb)
if (!index_exists(x, "omdb")) invisible(docs_bulk(x, omdb))

# can be sent without a body
docs_update_by_query(x, index='omdb')

# update
## note this works with imdbRating, a float, but didn't seem to work
## with Metascore, a long
## See link above for Painless API reference
body <- '{
  "script": {
    "source": "ctx._source.imdbRating++",
    "lang": "painless"
  },
  "query": {
    "match": {
      "Rated": "R"
    }
  }
}'
Search(x, "omdb", q = "Rated:\"R\"", asdf=TRUE,
  source = c("Title", "Rated", "imdbRating"))$hits$hits
docs_update_by_query(x, index='omdb', body = body)
Search(x, "omdb", q = "Rated:\"R\"", asdf=TRUE,
  source = c("Title", "Rated", "imdbRating"))$hits$hits

## End(Not run)

---

Elasticsearch documents functions.

Description

Elasticsearch documents functions.

Details

There are five functions to work directly with documents.

• docs_get()

• docs_mget()

• docs_create()

• docs_delete()

• docs_bulk()

Examples

## Not run: 
# Get a document
# docs_get(index='plos', type='article', id=1)

# Get multiple documents
# docs_mget(index="shakespeare", type="line", id=c(9,10))

# Create a document
# docs_create(index='plos', type='article', id=35, body=list(id="12345", title="New title"))

# Delete a document
# docs_delete(index='plos', type='article', id=35)

# Bulk load documents
# plosdat <- system.file("examples", "plos_data.json", package = "elastic")
# docs_bulk(plosdat)

## End(Not run)

---

elastic

Description

An Elasticsearch R client.

About

This package gives you access to local or remote Elasticsearch databases.

Quick start

If you're connecting to a Elasticsearch server already running, skip ahead to Search

Install Elasticsearch (on OSX)

• Download zip or tar file from Elasticsearch see here for download: https://www.elastic.co/downloads/elasticsearch

• Unzip it: ⁠untar elasticsearch-2.3.5.tar.gz⁠

• Move it: ⁠sudo mv elasticsearch-2.3.5 /usr/local⁠ (replace version with your version)

• Navigate to /usr/local: cd /usr/local

• Add shortcut: ⁠sudo ln -s elasticsearch-2.3.5 elasticsearch⁠ (replace version with your version)

For help on other platforms, see https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html

Start Elasticsearch

• Navigate to elasticsearch: cd /usr/local/elasticsearch

• Start elasticsearch: bin/elasticsearch

Initialization

The function connect() is used before doing anything else to set the connection details to your remote or local elasticsearch store. The details created by connect() are written to your options for the current session, and are used by elastic functions.

Search

The main way to search Elasticsearch is via the Search() function. E.g.:

Search()

Security

Elasticsearch is insecure out of the box! If you are running Elasticsearch locally on your own machine without exposing a port to the outside world, no worries, but if you install on a server with a public IP address, take the necessary precautions. There are a few options:

• Shield - A paid product - so probably only applicable to enterprise users

• DIY security - there are a variety of techniques for securing your Elasticsearch. I collected a number of resources in a blog post at https://recology.info/2015/02/secure-elasticsearch/

Elasticsearch changes

As of Elasticsearch v2:

• You can no longer create fields with dots in the name.

• Type names may not start with a dot (other than the special .percolator type)

• Type names may not be longer than 255 characters

• Types may no longer be deleted

• Queries and filters have been merged - all filter clauses are now query clauses. Instead, query clauses can now be used in query context or in filter context. See examples in Search() or Search_uri()

index names

The following are illegal characters, and can not be used in index names or types: ⁠\\⁠, /, *, ⁠?⁠, <, >, |, ⁠,⁠ (comma). double quote and whitespace are also illegal.

Author(s)

Scott Chamberlain

---

Defunct functions in elastic

Description

• mlt(): The MLT API has been removed, use More Like This Query via Search()

• nodes_shutdown(): The _shutdown API has been removed. Instead, setup Elasticsearch to run as a service (see Running as a Service on Linux (https://www.elastic.co/guide/en/elasticsearch/reference/2.0/setup-service.html) or Running as a Service on Windows (https://www.elastic.co/guide/en/elasticsearch/reference/2.0/setup-service-win.html)) or use the -p command line option to write the PID to a file.

• index_status(): _status route for the index API has been removed. Replaced with the Indices Stats and Indices Recovery APIs.

• mapping_delete(): Elasticsearch dropped this route in their API. Instead of deleting a mapping, delete the index and recreate with a new mapping.

---

Explain a search query.

Description

Explain a search query.

Usage

explain(
  conn,
  index,
  id,
  type = NULL,
  source2 = NULL,
  fields = NULL,
  routing = NULL,
  parent = NULL,
  preference = NULL,
  source = NULL,
  q = NULL,
  df = NULL,
  analyzer = NULL,
  analyze_wildcard = NULL,
  lowercase_expanded_terms = NULL,
  lenient = NULL,
  default_operator = NULL,
  source_exclude = NULL,
  source_include = NULL,
  body = NULL,
  raw = FALSE,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
index	Only one index. Required
id	Document id, only one. Required
type	Only one document type, optional
source2	(logical) Set to TRUE to retrieve the _source of the document explained. You can also retrieve part of the document by using source_include & source_exclude (see Get API for more details). This matches the ⁠_source⁠ term, but we want to avoid the leading underscore.
fields	Allows to control which stored fields to return as part of the document explained.
routing	Controls the routing in the case the routing was used during indexing.
parent	Same effect as setting the routing parameter.
preference	Controls on which shard the explain is executed.
source	Allows the data of the request to be put in the query string of the url.
q	The query string (maps to the query_string query).
df	The default field to use when no field prefix is defined within the query. Defaults to _all field.
analyzer	The analyzer name to be used when analyzing the query string. Defaults to the analyzer of the _all field.
analyze_wildcard	(logical) Should wildcard and prefix queries be analyzed or not. Default: FALSE
lowercase_expanded_terms	Should terms be automatically lowercased or not. Default: TRUE
lenient	If set to true will cause format based failures (like providing text to a numeric field) to be ignored. Default: FALSE
default_operator	The default operator to be used, can be AND or OR. Defaults to OR.
source_exclude	A vector of fields to exclude from the returned source2 field
source_include	A vector of fields to extract and return from the source2 field
body	The query definition using the Query DSL. This is passed in the body of the request.
raw	If TRUE (default), data is parsed to list. If FALSE, then raw JSON.
...	Curl args passed on to crul::HttpClient

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/search-explain.html

Examples

## Not run: 
(x <- connect())

explain(x, index = "plos", id = 14, q = "title:Germ")

body <- '{
 "query": {
   "match": { "title": "Germ" }
 }
}'
explain(x, index = "plos", id = 14, body=body)

## End(Not run)

---

Field capabilities

Description

The field capabilities API allows to retrieve the capabilities of fields among multiple indices.

Usage

field_caps(conn, fields, index = NULL, ...)

Arguments

conn	an Elasticsearch connection object, see connect()
fields	A list of fields to compute stats for. required
index	Index name, one or more
...	Curl args passed on to crul::verb-GET

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/search-field-caps.html

See Also

field_stats()

Examples

## Not run: 
x <- connect()
x$ping()

if (x$es_ver() >= 540) {
  field_caps(x, fields = "speaker", index = "shakespeare")
}


## End(Not run)

---

Search field statistics

Description

Search field statistics

Usage

field_stats(
  conn,
  fields = NULL,
  index = NULL,
  level = "cluster",
  body = list(),
  raw = FALSE,
  asdf = FALSE,
  ...
)

Arguments

conn	an Elasticsearch connection object, see connect()
fields	A list of fields to compute stats for. optional
index	Index name, one or more
level	Defines if field stats should be returned on a per index level or on a cluster wide level. Valid values are 'indices' and 'cluster' (default)
body	Query, either a list or json
raw	(logical) Get raw JSON back or not
asdf	(logical) If TRUE, use fromJSON to parse JSON directly to a data.frame. If FALSE (Default), list output is given.
...	Curl args passed on to crul::HttpClient

Details

The field stats api allows you to get statistical properties of a field without executing a search, but looking up measurements that are natively available in the Lucene index. This can be useful to explore a dataset which you don't know much about. For example, this allows creating a histogram aggregation with meaningful intervals based on the min/max range of values.

The field stats api by defaults executes on all indices, but can execute on specific indices too.

Note

Deprecated in Elasticsearch versions equal to/greater than 5.4.0

References

https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-field-stats.html

See Also

field_caps()

Examples

## Not run: 
x <- connect()

if (gsub("\\.", "", x$ping()$version$number) < 500) {
  field_stats(x, body = '{ "fields": ["speaker"] }', index = "shakespeare")
  ff <- c("scientificName", "continent", "decimalLatitude", "play_name", 
    "speech_number")
  field_stats(x, "play_name")
  field_stats(x, "play_name", level = "cluster")
  field_stats(x, ff, level = "indices")
  field_stats(x, ff)
  field_stats(x, ff, index = c("gbif", "shakespeare"))

  # can also pass a body, just as with Search()
  # field_stats(x, body = list(fields = "rating")) # doesn't work
  field_stats(x, body = '{ "fields": ["scientificName"] }', index = "gbif")

  body <- '{
    "fields" : ["scientificName", "decimalLatitude"]
  }'
  field_stats(x, body = body, level = "indices", index = "gbif")
}

## End(Not run)

---

fielddata

Description

Deep dive on fielddata details

Details

Most fields are indexed by default, which makes them searchable. Sorting, aggregations, and accessing field values in scripts, however, requires a different access pattern from search.

Text fields use a query-time in-memory data structure called fielddata. This data structure is built on demand the first time that a field is used for aggregations, sorting, or in a script. It is built by reading the entire inverted index for each segment from disk, inverting the term-document relationship, and storing the result in memory, in the JVM heap.

fielddata is disabled on text fields by default. Fielddata can consume a lot of heap space, especially when loading high cardinality text fields. Once fielddata has been loaded into the heap, it remains there for the lifetime of the segment. Also, loading fielddata is an expensive process which can cause users to experience latency hits. This is why fielddata is disabled by default. If you try to sort, aggregate, or access values from a script on a text field, you will see this exception:

"Fielddata is disabled on text fields by default. Set fielddata=true on your_field_name in order to load fielddata in memory by uninverting the inverted index. Note that this can however use significant memory."

To enable fielddata on a text field use the PUT mapping API, for example mapping_create("shakespeare", body = '{ "properties": { "speaker": { "type": "text", "fielddata": true } } }')

You may get an error about update_all_types, in which case set update_all_types=TRUE in mapping_create, e.g.,

mapping_create("shakespeare", update_all_types=TRUE, body = '{ "properties": { "speaker": { "type": "text", "fielddata": true } } }')

See https://www.elastic.co/guide/en/elasticsearch/reference/current/fielddata.html#_enabling_fielddata_on_literal_text_literal_fields for more information.

---

Index templates

Description

Index templates allow you to define templates that will automatically be applied when new indices are created

Usage

index_template_put(
  conn,
  name,
  body = NULL,
  create = NULL,
  flat_settings = NULL,
  master_timeout = NULL,
  order = NULL,
  timeout = NULL,
  ...
)

index_template_get(conn, name = NULL, filter_path = NULL, ...)

index_template_exists(conn, name, ...)

index_template_delete(conn, name, ...)

Arguments

conn	an Elasticsearch connection object, see connect()
name	(character) The name of the template
body	(character/list) The template definition
create	(logical) Whether the index template should only be added if new or can also replace an existing one. Default: FALSE
flat_settings	(logical) Return settings in flat format. Default: FALSE
master_timeout	(integer) Specify timeout for connection to master
order	(integer) The order for this template when merging multiple matching ones (higher numbers are merged later, overriding the lower numbers)
timeout	(integer) Explicit operation timeout
...	Curl options. Or in percolate_list function, further args passed on to Search()
filter_path	(character) a regex for filtering output path, see example

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html

Examples

## Not run: 
(x <- connect())

body <- '{
  "template": "te*",
  "settings": {
    "number_of_shards": 1
  },
  "mappings": {
    "type1": {
      "_source": {
        "enabled": false
      },
      "properties": {
        "host_name": {
          "type": "keyword"
        },
        "created_at": {
          "type": "date",
          "format": "EEE MMM dd HH:mm:ss Z YYYY"
        }
      }
    }
  }
}'
index_template_put(x, "template_1", body = body)

# get templates
index_template_get(x)
index_template_get(x, "template_1")
index_template_get(x, c("template_1", "template_2"))
index_template_get(x, "template_*")
## filter path
index_template_get(x, "template_1", filter_path = "*.template")

# template exists
index_template_exists(x, "template_1")
index_template_exists(x, "foobar")

# delete a template
index_template_delete(x, "template_1")
index_template_exists(x, "template_1")

## End(Not run)

---

Index API operations

Description

Index API operations

Usage

index_get(
  conn,
  index = NULL,
  features = NULL,
  raw = FALSE,
  verbose = TRUE,
  ...
)

index_exists(conn, index, ...)

index_delete(conn, index, raw = FALSE, verbose = TRUE, ...)

index_create(conn, index = NULL, body = NULL, raw = FALSE, verbose = TRUE, ...)

index_recreate(
  conn,
  index = NULL,
  body = NULL,
  raw = FALSE,
  verbose = TRUE,
  ...
)

index_close(conn, index, ...)

index_open(conn, index, ...)

index_stats(
  conn,
  index = NULL,
  metric = NULL,
  completion_fields = NULL,
  fielddata_fields = NULL,
  fields = NULL,
  groups = NULL,
  level = "indices",
  ...
)

index_settings(conn, index = "_all", ...)

index_settings_update(conn, index = NULL, body, ...)

index_segments(conn, index = NULL, ...)

index_recovery(conn, index = NULL, detailed = FALSE, active_only = FALSE, ...)

index_optimize(
  conn,
  index = NULL,
  max_num_segments = NULL,
  only_expunge_deletes = FALSE,
  flush = TRUE,
  wait_for_merge = TRUE,
  ...
)

index_forcemerge(
  conn,
  index = NULL,
  max_num_segments = NULL,
  only_expunge_deletes = FALSE,
  flush = TRUE,
  ...
)

index_upgrade(conn, index = NULL, wait_for_completion = FALSE, ...)

index_analyze(
  conn,
  text = NULL,
  field = NULL,
  index = NULL,
  analyzer = NULL,
  tokenizer = NULL,
  filters = NULL,
  char_filters = NULL,
  body = list(),
  ...
)

index_flush(
  conn,
  index = NULL,
  force = FALSE,
  full = FALSE,
  wait_if_ongoing = FALSE,
  ...
)

index_clear_cache(
  conn,
  index = NULL,
  filter = FALSE,
  filter_keys = NULL,
  fielddata = FALSE,
  query_cache = FALSE,
  id_cache = FALSE,
  ...
)

index_shrink(conn, index, index_new, body = NULL, ...)

Arguments

conn	an Elasticsearch connection object, see connect()
index	(character) A character vector of index names
features	(character) A single feature. One of settings, mappings, or aliases
raw	If TRUE (default), data is parsed to list. If FALSE, then raw JSON.
verbose	If TRUE (default) the url call used printed to console.
...	Curl args passed on to crul::HttpClient
body	Query, either a list or json.
metric	(character) A character vector of metrics to display. Possible values: "_all", "completion", "docs", "fielddata", "filter_cache", "flush", "get", "id_cache", "indexing", "merge", "percolate", "refresh", "search", "segments", "store", "warmer".
completion_fields	(character) A character vector of fields for completion metric (supports wildcards)
fielddata_fields	(character) A character vector of fields for fielddata metric (supports wildcards)
fields	(character) Fields to add.
groups	(character) A character vector of search groups for search statistics.
level	(character) Return stats aggregated on "cluster", "indices" (default) or "shards"
detailed	(logical) Whether to display detailed information about shard recovery. Default: FALSE
active_only	(logical) Display only those recoveries that are currently on-going. Default: FALSE
max_num_segments	(character) The number of segments the index should be merged into. Default: "dynamic"
only_expunge_deletes	(logical) Specify whether the operation should only expunge deleted documents
flush	(logical) Specify whether the index should be flushed after performing the operation. Default: TRUE
wait_for_merge	(logical) Specify whether the request should block until the merge process is finished. Default: TRUE
wait_for_completion	(logical) Should the request wait for the upgrade to complete. Default: FALSE
text	The text on which the analysis should be performed (when request body is not used)
field	Use the analyzer configured for this field (instead of passing the analyzer name)
analyzer	The name of the analyzer to use
tokenizer	The name of the tokenizer to use for the analysis
filters	A character vector of filters to use for the analysis
char_filters	A character vector of character filters to use for the analysis
force	(logical) Whether a flush should be forced even if it is not necessarily needed ie. if no changes will be committed to the index.
full	(logical) If set to TRUE a new index writer is created and settings that have been changed related to the index writer will be refreshed.
wait_if_ongoing	If TRUE, the flush operation will block until the flush can be executed if another flush operation is already executing. The default is false and will cause an exception to be thrown on the shard level if another flush operation is already running.
filter	(logical) Clear filter caches
filter_keys	(character) A vector of keys to clear when using the filter_cache parameter (default: all)
fielddata	(logical) Clear field data
query_cache	(logical) Clear query caches
id_cache	(logical) Clear ID caches for parent/child
index_new	(character) an index name, required. only applies to index_shrink method

Details

index_analyze: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-analyze.html This method can accept a string of text in the body, but this function passes it as a parameter in a GET request to simplify.

index_flush: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-flush.html From the ES website: The flush process of an index basically frees memory from the index by flushing data to the index storage and clearing the internal transaction log. By default, Elasticsearch uses memory heuristics in order to automatically trigger flush operations as required in order to clear memory.

index_status: The API endpoint for this function was deprecated in Elasticsearch v1.2.0, and will likely be removed soon. Use index_recovery() instead.

index_settings_update: There are a lot of options you can change with this function. See https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html for all the options.

index settings: See https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html for the static and dynamic settings you can set on indices.

Mappings

The "keyword" type is not supported in Elasticsearch < v5. If you do use a mapping with "keyword" type in Elasticsearch < v5 index_create() should fail.

Author(s)

Scott Chamberlain [email protected]

References

https://www.elastic.co/guide/en/elasticsearch/reference/current/indices.html

Examples

## Not run: 
# connection setup
(x <- connect())

# get information on an index
index_get(x, index='shakespeare')
## this one is the same as running index_settings('shakespeare')
index_get(x, index='shakespeare', features='settings')
index_get(x, index='shakespeare', features='mappings')
index_get(x, index='shakespeare', features='alias')

# check for index existence
index_exists(x, index='shakespeare')
index_exists(x, index='plos')

# create an index
if (index_exists(x, 'twitter')) index_delete(x, 'twitter')
index_create(x, index='twitter')
if (index_exists(x, 'things')) index_delete(x, 'things')
index_create(x, index='things')
if (index_exists(x, 'plos')) index_delete(x, 'plos')
index_create(x, index='plos')

# re-create an index
index_recreate(x, "deer")
index_recreate(x, "deer", verbose = FALSE)

# delete an index
if (index_exists(x, 'plos')) index_delete(x, index='plos')

## with a body
body <- '{
 "settings" : {
  "index" : {
    "number_of_shards" : 3,
    "number_of_replicas" : 2
   }
 }
}'
if (index_exists(x, 'alsothat')) index_delete(x, 'alsothat')
index_create(x, index='alsothat', body = body)
## with read only
body <- '{
 "settings" : {
  "index" : {
    "blocks" : {
      "read_only" : true
    }
   }
 }
}'
# index_create(x, index='myindex', body = body)
# then this delete call should fail with something like:
## > Error: 403 - blocked by: [FORBIDDEN/5/index read-only (api)]
# index_delete(x, index='myindex')

## with mappings
body <- '{
 "mappings": {
   "properties": {
     "location" : {"type" : "geo_point"}
   }
 }
}'
if (!index_exists(x, 'gbifnewgeo')) index_create(x, index='gbifnewgeo', body=body)
gbifgeo <- system.file("examples", "gbif_geosmall.json", package = "elastic")
docs_bulk(x, gbifgeo)

# close an index
index_create(x, 'plos')
index_close(x, 'plos')

# open an index
index_open(x, 'plos')

# Get stats on an index
index_stats(x, 'plos')
index_stats(x, c('plos','gbif'))
index_stats(x, c('plos','gbif'), metric='refresh')
index_stats(x, metric = "indexing")
index_stats(x, 'shakespeare', metric='completion')
index_stats(x, 'shakespeare', metric='completion', completion_fields = "completion")
index_stats(x, 'shakespeare', metric='fielddata')
index_stats(x, 'shakespeare', metric='fielddata', fielddata_fields = "evictions")
index_stats(x, 'plos', level="indices")
index_stats(x, 'plos', level="cluster")
index_stats(x, 'plos', level="shards")

# Get segments information that a Lucene index (shard level) is built with
index_segments(x)
index_segments(x, 'plos')
index_segments(x, c('plos','gbif'))

# Get recovery information that provides insight into on-going index shard recoveries
index_recovery(x)
index_recovery(x, 'plos')
index_recovery(x, c('plos','gbif'))
index_recovery(x, "plos", detailed = TRUE)
index_recovery(x, "plos", active_only = TRUE)

# Optimize an index, or many indices
if (x$es_ver() < 500) {
  ### ES < v5 - use optimize
  index_optimize(x, 'plos')
  index_optimize(x, c('plos','gbif'))
  index_optimize(x, 'plos')
} else {
  ### ES > v5 - use forcemerge
  index_forcemerge(x, 'plos')
}

# Upgrade one or more indices to the latest format. The upgrade process converts any
# segments written with previous formats.
if (x$es_ver() < 500) {
  index_upgrade(x, 'plos')
  index_upgrade(x, c('plos','gbif'))
}

# Performs the analysis process on a text and return the tokens breakdown
# of the text
index_analyze(x, text = 'this is a test', analyzer='standard')
index_analyze(x, text = 'this is a test', analyzer='whitespace')
index_analyze(x, text = 'this is a test', analyzer='stop')
index_analyze(x, text = 'this is a test', tokenizer='keyword',
  filters='lowercase')
index_analyze(x, text = 'this is a test', tokenizer='keyword',
  filters='lowercase', char_filters='html_strip')
index_analyze(x, text = 'this is a test', index = 'plos',
  analyzer="standard")
index_analyze(x, text = 'this is a test', index = 'shakespeare',
  analyzer="standard")

## NGram tokenizer
body <- '{
        "settings" : {
             "analysis" : {
                 "analyzer" : {
                     "my_ngram_analyzer" : {
                         "tokenizer" : "my_ngram_tokenizer"
                     }
                 },
                 "tokenizer" : {
                     "my_ngram_tokenizer" : {
                         "type" : "nGram",
                         "min_gram" : "2",
                         "max_gram" : "3",
                         "token_chars": [ "letter", "digit" ]
                     }
                 }
             }
      }
}'
if (index_exists(x, "shakespeare2")) index_delete(x, "shakespeare2")
tokenizer_set(x, index = "shakespeare2", body=body)
index_analyze(x, text = "art thouh", index = "shakespeare2",
  analyzer='my_ngram_analyzer')

# Explicitly flush one or more indices.
index_flush(x, index = "plos")
index_flush(x, index = "shakespeare")
index_flush(x, index = c("plos","shakespeare"))
index_flush(x, index = "plos", wait_if_ongoing = TRUE)
index_flush(x, index = "plos", verbose = TRUE)

# Clear either all caches or specific cached associated with one ore more indices.
index_clear_cache(x)
index_clear_cache(x, index = "plos")
index_clear_cache(x, index = "shakespeare")
index_clear_cache(x, index = c("plos","shakespeare"))
index_clear_cache(x, filter = TRUE)

# Index settings
## get settings
index_settings(x)
index_settings(x, "_all")
index_settings(x, 'gbif')
index_settings(x, c('gbif','plos'))
index_settings(x, '*s')
## update settings
if (index_exists(x, 'foobar')) index_delete(x, 'foobar')
index_create(x, "foobar")
settings <- list(index = list(number_of_replicas = 4))
index_settings_update(x, "foobar", body = settings)
index_get(x, "foobar")$foobar$settings

# Shrink index - Can only shrink an index if it has >1 shard
## index must be read only, a copy of every shard in the index must
## reside on the same node, and the cluster health status must be green
### index_settings_update call to change these
settings <- list(
  index.routing.allocation.require._name = "shrink_node_name",
  index.blocks.write = "true"
)
if (index_exists(x, 'barbarbar')) index_delete(x, 'barbarbar')
index_create(x, "barbarbar")
index_settings_update(x, "barbarbar", body = settings)
cat_recovery(x, index='barbarbar')
# index_shrink(x, "barbarbar", "barfoobbar")

## End(Not run)

---