Elasticsearch: Upsert with script
Update elasticsearch document with scripted upsert.
Let's say we have an elastisearch index, orders
, where we store a list of orders for a user. For example:
{
"username": "user1",
"orders": [
{
"id": "order1",
"timestamp": "2022-01-01T00:00:00Z",
"items": [
{
"id": "item1",
"name": "item1",
"price": 10,
"quantity": 1
},
{
"id": "item2",
"name": "item2",
"price": 20,
"quantity": 2
}
]
},
{
"id": "order2",
"timestamp": "2022-01-02T00:00:00Z",
"items": [
{
"id": "item3",
"name": "item3",
"price": 30,
"quantity": 3
},
{
"id": "item4",
"name": "item4",
"price": 40,
"quantity": 4
}
]
}
]
}
One assumption here: we will be using the username as document id
.
When updating a user's order, it can be done in two ways:
Option 1
Calling GET index API to check if that document exists. If exists, then update. If not create the document for that user.
We have this order data for user1
:
{
"username": "user1",
"order": {
"id": "order1",
"timestamp": "2022-01-01T00:00:00Z",
"items": [
{ "id": "item1", "name": "item1", "price": 10, "quantity": 1 },
{ "id": "item2", "name": "item2", "price": 20, "quantity": 2 }
]
}
}
But we don't know if there's already a record exists for user1
. Here's a python code snippet that first checks if the record exists, and if exists append the order to the users orders
array. If not create a new document for the user.
import json
from elasticsearch import Elasticsearch
from elasticsearch.exceptions import NotFoundError
es = Elasticsearch(hosts=["http://localhost:9200"])
def update_or_insert(body):
doc_id = body["username"]
try:
res = es.get(index="orders", id=doc_id)
# Update
updated_body = {
"script": {
"source": "ctx._source.orders.add(params.order)",
"lang": "painless",
"params": {"order": body["order"]}
}
}
es.update(index="orders", id=doc_id, body=updated_body)
except NotFoundError:
res = es.index(index="orders", id=doc_id, body=body)
return res
if __name__ == "__main__":
data = {
"username": "user1",
"order": {
"id": "order1",
"timestamp": "2022-01-01T00:00:00Z",
"items": [
{"id": "item1", "name": "item1", "price": 10, "quantity": 1},
{"id": "item2", "name": "item2", "price": 20, "quantity": 2},
],
},
}
print(update_or_insert(data))
Option 2
Use upsert, which will create the document if not exists or update if exists, which is one single API call.
Here's a sample python code that demonstrate the upsert operation:
import json
from elasticsearch import Elasticsearch
from elasticsearch.exceptions import NotFoundError
es = Elasticsearch(hosts=["http://localhost:9200"])
def upsert(body):
updated_body = {
"script": {
"source": "ctx._source.orders.add(params.order)",
"lang": "painless",
"params": {"order": body["order"]}
},
"upsert": {
"username": body["username"],
"orders": [body["order"]]
},
"scripted_upsert": True
}
res = es.update(index="orders", id=body["username"], body=updated_body)
return res
if __name__ == "__main__":
data = {
"username": "user1",
"order": {
"id": "order1",
"timestamp": "2022-01-01T00:00:00Z",
"items": [
{"id": "item1", "name": "item1", "price": 10, "quantity": 1},
{"id": "item2", "name": "item2", "price": 20, "quantity": 2},
],
},
}
print(upsert(data))
Codes can be found in this repo: github.com/tanjibpa/es-upsert