Table of content
OpenSearch Serverless
Amazon OpenSearch Serverless (AOSS) is an on-demand serverless configuration for Amazon OpenSearch Service. Serverless removes the operational complexities of provisioning, configuring, and tuning your OpenSearch clusters. It’s a good option for organizations that don’t want to self-manage their OpenSearch clusters. With AOSS, you can easily search and analyze a large volume of data without having to worry about the underlying infrastructure and data management.
Collection
An AOSS collection is a group of OpenSearch indexes that work together to support a specific workload or use case.
Collections have the same kind of high-capacity, distributed, and highly available storage volume that’s used by provisioned OpenSearch Service domains. Data is encrypted in transit within a collection.
Choosing a collection type
You choose a collection type when you first create a collection. AOSS supports three primary collection types:
- Time series – The log analytics segment that focuses on analyzing large volumes of semi-structured, machine-generated data in real-time for operational, security, user behavior, and business insights.
- Search – Full-text search that powers applications in your internal networks (e.g. CMS) and internet-facing applications, such as ecommerce website search and content search.
- Vector search – Semantic search on vector embeddings that simplifies vector data management and powers machine learning (ML) augmented search experiences and generative AI applications, such as chatbots, personal assistants, and fraud detection.
Network Policy
Network policies let you manage many collections at scale by automatically assigning network access settings to collections that match the rules defined in the policy.
In a network policy, you specify a series of rules. These rule define access permissions to collection endpoints and OpenSearch Dashboards endpoints. Each rule consists of an access type (public or VPC) and a resource type (collection and/or OpenSearch Dashboards endpoint). For each resource type (collection and dashboard), you specify a series of rules that define which collection(s) the policy will apply to.
Data Access Policy
Data access policies define how your users access the data within your collections. Data access policies help you manage collections at scale by automatically assigning access permissions to collections and indexes that match a specific pattern. Multiple policies can apply to a single resource.
Implementation using CDK
This diagram illustrates a straightforward scenario. When a Lambda function is situated within a Virtual Private Cloud (VPC), it serves as a client to OpenSearch. In this setup, the Lambda function can connect to OpenSearch through a VPC Endpoint. Conversely, if the Lambda function is not hosted within a VPC, the use of a VPC Endpoint for connectivity to OpenSearch is not necessary.
componentDiagram
title: Architecture Diagram
[Lambda (in VPC)] -- SecurityGroup-Outgoing
SecurityGroup-Outgoing -- SecurityGroup-Incoming
SecurityGroup-Incoming --> [VPC Endpoint]
[VPC Endpoint] --> [OpenSearchServerless]
Stack
import * as cdk from "aws-cdk-lib";
import { Construct } from "constructs";
import { CDKContext } from "../types";
import { VpcConstruct } from "./constructs/vpc";
import { IamConstruct } from "./constructs/iam";
import { LambdaConstruct } from "./constructs/lambda";
import { OpenSearchConstruct } from "./constructs/opensearch";
export class Stack extends cdk.Stack {
constructor(
scope: Construct,
id: string,
ctx: CDKContext,
props?: cdk.StackProps
) {
super(scope, id, props);
const vpcConstruct = new VpcConstruct(this, "aoss-vpc", ctx);
const executorRole = new IamConstruct(this, "aoss-iam-role", ctx);
const aossConstruct = new OpenSearchConstruct(
this,
"OpenSearchConstruct",
ctx,
{ vpcConstruct, executorRole: executorRole.role }
);
new LambdaConstruct(this, "aoss-lambda", { vpcConstruct, aossConstruct });
}
}VPC
import { Construct } from "constructs";
import { IVpc, Port, SecurityGroup, Vpc } from "aws-cdk-lib/aws-ec2";
import { CDKContext } from "../../types";
export class VpcConstruct extends Construct {
public readonly vpc: IVpc;
public readonly vpcEndpointSg: SecurityGroup;
public readonly lambdaSg: SecurityGroup;
constructor(scope: Construct, id: string, ctx: CDKContext) {
super(scope, id);
this.vpc = Vpc.fromLookup(this, "vpc", {
vpcId: ctx.resources.vpc.vpcId,
region: ctx.region,
ownerAccountId: ctx.accountId,
});
this.vpcEndpointSg = new SecurityGroup(this, "aoss-vpc-endpoint-sg", {
description: "aoss-vpc-endpoint-sg",
securityGroupName: "aoss-vpc-endpoint-sg",
vpc: this.vpc,
});
this.lambdaSg = new SecurityGroup(this, "aoss-lambda-sg", {
description: "aoss-lambda-sg",
securityGroupName: "aoss-lambda-sg",
vpc: this.vpc,
allowAllOutbound: false,
});
this.lambdaSg.addEgressRule(this.vpcEndpointSg, Port.allTraffic());
this.vpcEndpointSg.addIngressRule(this.lambdaSg, Port.allTraffic());
const eksClusterSg = SecurityGroup.fromLookupById(
this,
"eks-cluster-sg",
ctx.resources.eks.clusterSecurityGroupId
);
this.vpcEndpointSg.addIngressRule(eksClusterSg, Port.allTraffic());
}
}Lambda
import {
Effect,
ManagedPolicy,
PolicyStatement,
Role,
ServicePrincipal,
} from "aws-cdk-lib/aws-iam";
import {
NodejsFunction,
NodejsFunctionProps,
} from "aws-cdk-lib/aws-lambda-nodejs";
import { join } from "path";
import { RetentionDays } from "aws-cdk-lib/aws-logs";
import * as lambda from "aws-cdk-lib/aws-lambda";
import { Construct } from "constructs";
import { VpcConstruct } from "./vpc";
import { OpenSearchConstruct } from "./opensearch";
import { Aws } from "aws-cdk-lib";
interface LambdaConstructProps {
vpcConstruct: VpcConstruct;
aossConstruct: OpenSearchConstruct;
}
export class LambdaConstruct extends Construct {
public readonly nodejsFunction: NodejsFunction;
constructor(
scope: Construct,
id: string,
private props: LambdaConstructProps
) {
super(scope, id);
this.nodejsFunction = new NodejsFunction(
this,
"aossLambdaApiExecutorFunction",
{
entry: join(__dirname, "..", "..", "handlers", "aoss-lambda.ts"),
functionName: "aoss-lambda-api-executor",
role: this.lambdaRole(),
...this.nodeJsFunctionProps,
logRetention: RetentionDays.ONE_WEEK,
environment: {
COLLECTION_ENDPOINT:
this.props.aossConstruct.collection.attrCollectionEndpoint,
},
vpc: this.props.vpcConstruct.vpc,
vpcSubnets: { subnets: this.props.vpcConstruct.vpc.privateSubnets },
securityGroups: [this.props.vpcConstruct.lambdaSg],
}
);
}
private lambdaRole() {
return new Role(this, "aossLambdaApiExecutorRole", {
roleName: `aoss-lambda-api-executor-role`,
assumedBy: new ServicePrincipal("lambda.amazonaws.com"),
managedPolicies: [
ManagedPolicy.fromAwsManagedPolicyName(
"service-role/AWSLambdaBasicExecutionRole"
),
ManagedPolicy.fromAwsManagedPolicyName(
"AmazonOpenSearchServiceFullAccess"
),
new ManagedPolicy(this, "lambda-basic-execution-policy", {
managedPolicyName: "lambda-basic-execution-policy",
statements: [
new PolicyStatement({
effect: Effect.ALLOW,
actions: ["logs:CreateLogGroup"],
resources: [`arn:aws:logs::${Aws.ACCOUNT_ID}:*`],
}),
new PolicyStatement({
effect: Effect.ALLOW,
actions: ["logs:CreateLogStream", "logs:PutLogEvents"],
resources: [
`arn:aws:logs::${Aws.ACCOUNT_ID}:log-group:/aws/lambda/aoss-lambda-api-executor:*`,
],
}),
],
}),
new ManagedPolicy(this, "vpc-access-execution-policy", {
managedPolicyName: "vpc-access-execution-policy",
statements: [
new PolicyStatement({
effect: Effect.ALLOW,
actions: [
"ec2:CreateNetworkInterface",
"ec2:DeleteNetworkInterface",
"ec2:DescribeNetworkInterfaces",
],
resources: ["*"],
}),
],
}),
new ManagedPolicy(this, "aoss-full-api-access", {
managedPolicyName: "aoss-full-api-access",
statements: [
new PolicyStatement({
effect: Effect.ALLOW,
actions: ["aoss:BatchGetCollection", "aoss:APIAccessAll"],
resources: ["*"],
}),
],
}),
],
});
}
private nodeJsFunctionProps: NodejsFunctionProps = {
bundling: { externalModules: ["aws-sdk"] },
runtime: lambda.Runtime.NODEJS_18_X,
};
}IAM Role
import { Construct } from "constructs";
import {
AccountPrincipal,
Effect,
FederatedPrincipal,
ManagedPolicy,
PolicyStatement,
Role,
ServicePrincipal,
} from "aws-cdk-lib/aws-iam";
import { CDKContext } from "../../types";
export class IamConstruct extends Construct {
public readonly role: Role;
constructor(scope: Construct, id: string, ctx: CDKContext) {
super(scope, id);
this.role = new Role(this, "aossExecutorRole", {
roleName: `aoss-executor-role`,
assumedBy: new ServicePrincipal("eks.amazonaws.com"),
managedPolicies: [
ManagedPolicy.fromAwsManagedPolicyName(
"AmazonOpenSearchServiceFullAccess"
),
new ManagedPolicy(this, "vpc-access-policy", {
managedPolicyName: "eks-vpc-access-policy",
statements: [
new PolicyStatement({
effect: Effect.ALLOW,
actions: [
"ec2:CreateNetworkInterface",
"ec2:DeleteNetworkInterface",
"ec2:DescribeNetworkInterfaces",
],
resources: ["*"],
}),
],
}),
new ManagedPolicy(this, "eks-aoss-api-access1", {
managedPolicyName: "eks-aoss-api-access1",
statements: [
new PolicyStatement({
effect: Effect.ALLOW,
actions: ["aoss:BatchGetCollection", "aoss:APIAccessAll"],
resources: ["*"],
}),
],
}),
],
});
this.role.assumeRolePolicy?.addStatements(
new PolicyStatement({
effect: Effect.ALLOW,
principals: [new AccountPrincipal(ctx.accountId)],
actions: ["sts:AssumeRole"],
}),
new PolicyStatement({
effect: Effect.ALLOW,
principals: [
new FederatedPrincipal(
`arn:aws:iam::${ctx.accountId}:oidc-provider/oidc.eks.${ctx.region}.amazonaws.com/id/${ctx.resources.eks.oidcId}`
),
],
actions: ["sts:AssumeRoleWithWebIdentity"],
conditions: {
StringEquals: {
[`oidc.eks.${ctx.region}.amazonaws.com/id/${ctx.resources.eks.oidcId}:sub`]: `system:serviceaccount:${ctx.resources.eks.namespace}:${ctx.resources.eks.serviceAccountName}`,
},
},
})
);
}
}OpenSearch Serverless
import { Construct } from "constructs";
import {
CfnAccessPolicy,
CfnCollection,
CfnSecurityPolicy,
CfnVpcEndpoint,
} from "aws-cdk-lib/aws-opensearchserverless";
import { VpcConstruct } from "./vpc";
import { Role } from "aws-cdk-lib/aws-iam";
import { CDKContext } from "../../types";
interface OpenSearchConstructProps {
vpcConstruct: VpcConstruct;
executorRole: Role;
}
export class OpenSearchConstruct extends Construct {
public readonly searchDomain: string;
public readonly collection: CfnCollection;
constructor(
scope: Construct,
id: string,
ctx: CDKContext,
props: OpenSearchConstructProps
) {
super(scope, id);
this.collection = new CfnCollection(this, "MySearchCollection", {
name: "my-collection",
type: "SEARCH",
});
const encPolicy = new CfnSecurityPolicy(this, "MySecurityPolicy", {
name: "my-collection-policy",
policy:
'{"Rules":[{"ResourceType":"collection","Resource":["collection/my-collection"]}],"AWSOwnedKey":true}',
type: "encryption",
});
this.collection.addDependency(encPolicy);
const vpcEndpoint = new CfnVpcEndpoint(this, "oss-vpce", {
name: "oss-vpce",
securityGroupIds: [props.vpcConstruct.vpcEndpointSg.securityGroupId],
subnetIds: [
...props.vpcConstruct.vpc.privateSubnets.map(i => i.subnetId),
], // either private or isolated
vpcId: props.vpcConstruct.vpc.vpcId,
});
// Network policy is required so that the dashboard can be viewed!
const netPolicy = new CfnSecurityPolicy(this, "ProductNetworkPolicy", {
name: "my-network-policy",
policy: JSON.stringify([
{
Rules: [
{
ResourceType: "collection",
Resource: ["collection/my-collection"],
},
{
ResourceType: "dashboard",
Resource: ["collection/my-collection"],
},
],
AllowFromPublic: false,
SourceVPCEs: [vpcEndpoint.ref],
},
{
Rules: [
{
ResourceType: "collection",
Resource: ["collection/my-collection"],
},
{
ResourceType: "dashboard",
Resource: ["collection/my-collection"],
},
],
AllowFromPublic: true,
},
]),
type: "network",
});
this.collection.addDependency(netPolicy);
// Data collection policy
const dataAccessPolicy = new CfnAccessPolicy(this, "MyCfnAccessPolicy", {
name: "my-data-access-policy",
type: "data",
policy: JSON.stringify([
{
Rules: [
{
Resource: ["collection/my-collection"],
Permission: [
"aoss:CreateCollectionItems",
"aoss:DeleteCollectionItems",
"aoss:UpdateCollectionItems",
"aoss:DescribeCollectionItems",
],
ResourceType: "collection",
},
{
Resource: ["index/my-collection/*"],
Permission: [
"aoss:CreateIndex",
"aoss:DeleteIndex",
"aoss:UpdateIndex",
"aoss:DescribeIndex",
"aoss:ReadDocument",
"aoss:WriteDocument",
], // TODO: Update this as per the requirement
ResourceType: "index",
},
],
Principal: [
`arn:aws:iam::${ctx.accountId}:role/aoss-lambda-api-executor-role`,
props.executorRole.roleArn,
`arn:aws:iam::${ctx.accountId}:role/AnyOtherRole`,
],
},
]),
});
this.collection.addDependency(dataAccessPolicy);
this.searchDomain = this.collection.attrCollectionEndpoint;
}
}