Skip to main content

How to configure OpenSearch S3 snapshots with ClusterNest

OpenSearch supports backing up and restoring data to AWS S3 or an S3-compatible storage by configuring a snapshot repository.

For authentication to S3, ClusterNest managed OpenSearch supports both IAM Roles and S3 Access Keys. If you are using AWS S3 the preferred method is to use the IAM Role method since it does not require any static credentials.

Using AWS IAM Role​

To utilize IAM role for S3 auth. Enable the IAM Role attachment on the ClusterNest managed OpenSearch Cluster. This would assign an IAM Role to the OpenSearch cluster which can be granted access to your bucket with a cross-account bucket policy.

The Terraform code below will:

  1. Configure the ClusterNest provider
  2. Create an OpenSearch cluster with IAM Role enabled in the repo_config
  3. Provision an S3 bucket with ACLs and a cross account policy
  4. Fetch OpenSearch credentials
  5. Configure the OpenSearch provider
  6. Register the snapshot repository with your S3 bucket
terraform {
  required_providers {
    clusternest = {
      source  = "tf.clusternest.com/clusternest/clusternest"
      version = "1.0.3"
    }
    opensearch = {
      source  = "opensearch-project/opensearch"
      version = ">= 2.2.0"
    }
    aws = {
      source  = "hashicorp/aws"
      version = "6.5.0"
    }
  }
}

provider "clusternest" {
  email    = "[email protected]"
  password = "app-password"
}

resource "clusternest_opensearch" "test" {
  name            = "repo-config"
  tier            = "standard"
  organization_id = 123
  repo_config = {
    enable_iam_role = true
  }
}

data "clusternest_opensearch_credentials" "test" {
  cluster_id = clusternest_opensearch.test.id
}

provider "opensearch" {
  url      = clusternest_opensearch.test.opensearch_url
  username = data.clusternest_opensearch_credentials.test.username
  password = data.clusternest_opensearch_credentials.test.password
}

resource "aws_s3_bucket" "example" {
  bucket = "opensearch-snapshot-example"
}

resource "aws_s3_bucket_ownership_controls" "example" {
  bucket = aws_s3_bucket.example.id
  rule {
    object_ownership = "BucketOwnerPreferred"
  }
}

resource "aws_s3_bucket_acl" "example" {
  depends_on = [aws_s3_bucket_ownership_controls.example]
  bucket     = aws_s3_bucket.example.id
  acl        = "private"
}

resource "aws_s3_bucket_policy" "allow_access_from_another_account" {
  bucket = aws_s3_bucket.example.id
  policy = data.aws_iam_policy_document.allow_access_from_another_account.json
}

data "aws_iam_policy_document" "allow_access_from_another_account" {
  statement {
    principals {
      type        = "AWS"
      identifiers = [clusternest_opensearch.test.repo_config.aws_role_arn]
    }
    actions = [
      "s3:GetBucketLocation",
      "s3:ListBucket",
      "s3:ListBucketMultipartUploads",
      "s3:ListBucketVersions"
    ]
    resources = [aws_s3_bucket.example.arn]
  }
  statement {
    principals {
      type        = "AWS"
      identifiers = [clusternest_opensearch.test.repo_config.aws_role_arn]
    }
    actions = [
      "s3:AbortMultipartUpload",
      "s3:DeleteObject",
      "s3:GetObject",
      "s3:ListMultipartUploadParts",
      "s3:PutObject"
    ]
    resources = ["${aws_s3_bucket.example.arn}/*"]
  }
}

resource "opensearch_snapshot_repository" "repo" {
  name     = "es-index-backups"
  type     = "s3"
  settings = {
    bucket   = aws_s3_bucket.example.id
    region   = aws_s3_bucket.example.bucket_region
    endpoint = trimprefix(aws_s3_bucket.example.bucket_regional_domain_name, "${aws_s3_bucket.example.id}.")
  }
}