Attention

For production workloads, we recommend to create the buckets separately for any bucket that stores critical data. See Lookup to use existing buckets.

x-s3

x-s3:
  bucket:
    Properties: {}
    MacroParameters: {}
    Lookup: {}
    Settings: {}
    Services: {}

Define or use existing S3 buckets to use with your services or other AWS Resources (where applicable).

Hint

When the bucket uses encryption, ECS Compose-X will automatically identify the KMS key and if applicable, grant the necessary permissions to the service role. The KMS predefined EncryptDecrypt policy will be used to that effect.

Services

Model

Services:
  service01:
    Access:
      bucket: <>
      objects: <>
    ReturnValues: {}

As for all other resource types, you can define the type of access you want based to the S3 buckets. However, for buckets, this means distinguish the bucket and the objects resource.

Short example
x-s3:
  bucketA:
    Properties: {}
    Settings: {}
    Services:
      service-01:
        Access:
          objects: RW
          bucket: ListOnly

services:
  service-01: {}

IAM Permissions

For S3 buckets, the access types is expecting a object with objects and bucket to distinguish permissions for each. If you indicate a string, the default permissions (bucket: ListOnly and objects: RW) will be applied.

Full access types policies definitions
{
  "objects": {
    "CRUD": {
      "Action": [
        "s3:GetObject",
        "s3:DeleteObject",
        "s3:PutObject",
        "s3:GetObjectTagging",
        "s3:GetObjectVersionTagging",
        "s3:PutObjectTagging",
        "s3:PutObjectVersionTagging",
        "s3:DeleteObjectTagging",
        "s3:DeleteObjectVersionTagging",
        "s3:PutObjectAcl",
        "s3:AbortMultipartUpload",
        "s3:CreateMultipartUpload"
      ],
      "Effect": "Allow",
      "Resource": [
        "${ARN}/*"
      ]
    },
    "RW": {
      "Action": [
        "s3:GetObject*",
        "s3:PutObject*"
      ],
      "Effect": "Allow",
      "Resource": [
        "${ARN}/*"
      ]
    },
    "StrictRW": {
      "Action": [
        "s3:GetObject",
        "s3:PutObject"
      ],
      "Effect": "Allow",
      "Resource": [
        "${ARN}/*"
      ]
    },
    "StrictRWDelete": {
      "Action": [
        "s3:GetObject",
        "s3:PutObject",
        "s3:DeleteObject"
      ],
      "Effect": "Allow",
      "Resource": [
        "${ARN}/*"
      ]
    },
    "RWDelete": {
      "Action": [
        "s3:GetObject*",
        "s3:PutObject*",
        "s3:DeleteObject*"
      ],
      "Effect": "Allow",
      "Resource": [
        "${ARN}/*"
      ]
    },
    "ReadOnly": {
      "Action": [
        "s3:GetObject*"
      ],
      "Effect": "Allow",
      "Resource": [
        "${ARN}/*"
      ]
    },
    "StrictReadOnly": {
      "Action": [
        "s3:GetObject"
      ],
      "Effect": "Allow",
      "Resource": [
        "${ARN}/*"
      ]
    },
    "WriteOnly": {
      "Action": [
        "s3:PutObject*"
      ],
      "Effect": "Allow",
      "Resource": [
        "${ARN}/*"
      ]
    },
    "StrictWriteOnly": {
      "Action": [
        "s3:PutObject"
      ],
      "Effect": "Allow",
      "Resource": [
        "${ARN}/*"
      ]
    }
  },
  "bucket": {
    "ListOnly": {
      "Effect": "Allow",
      "Action": [
        "s3:ListBucket",
        "s3:GetBucketLocation",
        "s3:GetBucketPublicAccessBlock"
      ],
      "Resource": [
        "${ARN}"
      ]
    },
    "PowerUser": {
      "Effect": "Allow",
      "Action": [
        "s3:ListBucket",
        "s3:GetBucket*",
        "s3:SetBucket*"
      ],
      "Resource": [
        "${ARN}"
      ]
    }
  },
  "enforceSecureConnection": {
    "enforceSecureConnection": {
      "Sid": "AllowSSLRequestsOnly",
      "Action": "s3:*",
      "Effect": "Deny",
      "Condition": {
        "Bool": {
          "aws:SecureTransport": "false"
        }
      },
      "Resource": [
        "${ARN}",
        "${ARN}/*"
      ]
    }
  },
  "PredefinedBucketPolicies": {
    "enforceSecureConnection": {
      "Sid": "AllowSSLRequestsOnly",
      "Principal": {
        "AWS": "*"
      },
      "Action": "s3:*",
      "Effect": "Deny",
      "Condition": {
        "Bool": {
          "aws:SecureTransport": "false"
        }
      },
      "Resource": [
        "${ARN}",
        "${ARN}/*"
      ]
    }
  },
  "kinesis_firehose": {
    "s3destination": {
      "Effect": "Allow",
      "Action": [
        "s3:AbortMultipartUpload",
        "s3:GetBucketLocation",
        "s3:GetObject",
        "s3:ListBucket",
        "s3:ListBucketMultipartUploads",
        "s3:PutObject",
        "s3:PutObjectAcl"
      ],
      "Resource": [
        "${ARN}",
        "${ARN}/*"
      ]
    },
    "s3keyaccess": {
      "Effect": "Allow",
      "Action": [
        "kms:Decrypt",
        "kms:GenerateDataKey"
      ],
      "Resource": [
        "${ARN}"
      ]
    }
  }
}

ReturnValues

For full details, see AWS S3 Return Values for available options.

The return value BucketName can be used to return the value of the Ref Function.

Warning

If you return values that rely on features you have not enabled, i.e. WebsiteURL , the stack creation / update will fail.

Properties

For the properties, go to to AWS CFN S3 Definition We highly encourage not to set the bucket name there. If you did, we recommend to use the ExpandRegionToBucket and ExpandAccountIdToBucket to make the bucket unique.

MacroParameters

Some use-cases require special adjustments. This is what this section is for.

NameSeparator

Default is - which separates the different parts of the bucket that you might have automatically added via the other MacroParameters

As shown below, the separator between the bucket name and AWS::AccountId or AWS::Region is - . This parameter allows you to define something else.

Note

I would recommend not more than 2 characters separator.

Warning

The separator must allow for DNS compliance [a-z0-9.-]

ExpandRegionToBucket

When definining the BucketName in properties, if wanted to, for uniqueness or readability, you can append to that string the region id (which is DNS compliant) to the bucket name.

Properties:
  BucketName: abcd-01
Settings:
  ExpandRegionToBucket: True

Results into

!Sub abcd-01-${AWS::Region}

ExpandAccountIdToBucket

Similar to ExpandRegionToBucket, it will append the account ID (additional or instead of).

Properties:
  BucketName: abcd-01
Settings:
  ExpandRegionToBucket: True

Results into

!Sub 'abcd-01-${AWS::AccountId}'

Hint

If you set both ExpandAccountIdToBucket and ExpandRegionToBucket, you end up with

!Sub 'abcd-01-${AWS::Region}-${AWS::AccountId}'

Lookup

Refer to Lookup for the full details.

x-s3:
  existing-bucket:
    Lookup:
      Tags:
        - name: my-first-bucket
        - environment: dev

Examples

Create new S3 buckets
version: "3.8"

x-s3:
  DeletionPolicy: Retain
  bucket-01:
    Properties:
      BucketName: bucket-01
      AccessControl: BucketOwnerFullControl
      ObjectLockEnabled: True
      PublicAccessBlockConfiguration:
          BlockPublicAcls: True
          BlockPublicPolicy: True
          IgnorePublicAcls: True
          RestrictPublicBuckets: False
      AccelerateConfiguration:
        AccelerationStatus: Suspended
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: "aws:kms"
              KMSMasterKeyID: "aws/s3"
      VersioningConfiguration:
        Status: "Enabled"
    MacroParameters:
      ExpandRegionToBucket: True
      ExpandAccountIdToBucket: True
    Services:
      - name: app03
        access:
          bucket: ListOnly
          objects: CRUD
  bucket-03:
    Properties:
      BucketName: bucket-03
      AccessControl: BucketOwnerFullControl
      ObjectLockEnabled: True
      PublicAccessBlockConfiguration:
          BlockPublicAcls: True
          BlockPublicPolicy: True
          IgnorePublicAcls: True
          RestrictPublicBuckets: False
      AccelerateConfiguration:
        AccelerationStatus: Suspended
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
      VersioningConfiguration:
        Status: "Enabled"

    Settings:
      ExpandRegionToBucket: True
      ExpandAccountIdToBucket: False
    Services:
      - name: app03
        access:
          bucket: ListOnly
          objects: CRUD
  bucket-02:
    Properties: {}
    Settings:
      ExpandRegionToBucket: False
      ExpandAccountIdToBucket: False
      EnableEncryption: AES256
      EnableAcceleration: True
    Services:
      - name: app03
        access:
          bucket: ListOnly
          objects: RW

  bucket-04:
    Properties:
      BucketName: bucket-04
    Settings:
      NameSeparator: "."
      ExpandRegionToBucket: False
      ExpandAccountIdToBucket: False
      EnableEncryption: AES256
      EnableAcceleration: True
    Services:
      - name: app03
        access:
          bucket: ListOnly
          objects: RW
    MacroParameters:
      BucketPolicy:
        PredefinedBucketPolicies:
          - enforceSecureConnection
        Policies:
          - Effect: Allow
            Action:
              - s3:Get*
            Resource:
              - "${!ARN}/*"
          - Effect: Allow
            Action:
              - s3:Get*
              - s3:List*
            Resource:
              - "${!ARN}/*"
              - "${!ARN}"
            Condition:
              bool:
                aws:sourceIp: abcd
Lookup and use only existing buckets
version: "3.8"

x-s3:
  bucket-07:
    Lookup:
      Tags:
        - aws:cloudformation:logical-id: ArtifactsBucket
        - aws:cloudformation:stack-name: pipeline-shared-buckets
    Services:
      - name: app03
        access:
          bucket: PowerUser
          objects: RW

  bucket-08:
    Lookup:
      Identifier: sacrificial-lamb
      Tags:
        - composex: "True"
    Services:
      - name: app03
        access:
          bucket: PowerUser
          objects: RW
          enforceSecureConnection: true
Create new bucket with AWS CFN properties
version: "3.8"

x-s3:
  bucket-01:
    Properties:
      BucketName: bucket-01
      AccessControl: BucketOwnerFullControl
      AccelerateConfiguration:
        AccelerationStatus: Suspended
      ObjectLockEnabled: True
      PublicAccessBlockConfiguration:
        BlockPublicAcls: True
        BlockPublicPolicy: True
        IgnorePublicAcls: True
        RestrictPublicBuckets: False
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: "aws:kms"
              KMSMasterKeyID: "aws/s3"
      VersioningConfiguration:
        Status: "Enabled"
      MetricsConfigurations:
        - Id: EntireBucket
      LifecycleConfiguration:
        Rules:
          - Id: GlacierRule
            Prefix: glacier
            Status: Enabled
            ExpirationInDays: '365'
            Transitions:
              - TransitionInDays: '1'
                StorageClass: GLACIER
      CorsConfiguration:
        CorsRules:
          - AllowedHeaders:
              - '*'
            AllowedMethods:
              - GET
            AllowedOrigins:
              - '*'
            ExposedHeaders:
              - Date
            Id: myCORSRuleId1
            MaxAge: '3600'
          - AllowedHeaders:
              - x-amz-*
            AllowedMethods:
              - DELETE
            AllowedOrigins:
              - 'http://www.example.com'
              - 'http://www.example.net'
            ExposedHeaders:
              - Connection
              - Server
              - Date
            Id: myCORSRuleId2
            MaxAge: '1800'
      WebsiteConfiguration:
        IndexDocument: index.html
        ErrorDocument: error.html
        RoutingRules:
          - RoutingRuleCondition:
              HttpErrorCodeReturnedEquals: '404'
              KeyPrefixEquals: out1/
            RedirectRule:
              HostName: ec2-11-22-333-44.compute-1.amazonaws.com
              ReplaceKeyPrefixWith: report-404/
      NotificationConfiguration:
        TopicConfigurations:
          - Topic: 'arn:aws:sns:us-east-1:123456789012:TestTopic'
            Event: 's3:ReducedRedundancyLostObject'
    MacroParameters:
      ExpandRegionToBucket: True
      ExpandAccountIdToBucket: True
    Services:
      - name: app03
        access:
          objects: CRUD
          bucket: ListOnly
          s3-bucket-ssl-requests-only: true

      - name: app02
        access:
          objects: CRUD
          bucket: ListOnly

JSON Schema

Model

x-s3

x-s3.spec.json

x-s3 specification for ECS Cluster

type

object

properties

  • Lookup

x-resources.common.spec.json#/definitions/Lookup

  • Use

type

string

  • Properties

https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-s3-bucket.html

type

object

  • Settings

x-resources.common.spec.json#/definitions/Settings

  • MacroParameters

type

object

properties

  • BucketPolicy

type

object

properties

  • Statement

type

array

items

services.x-iam.spec.json#/definitions/Statement

  • PredefinedBucketPolicies

type

array

items

type

string

enum

enforceSecureConnection

uniqueItems

True

  • Services

#/definitions/ServicesDef

definitions

  • ServicesDef

oneOf

type

array

items

type

object

properties

  • name

type

string

  • access

#/definitions/ServicesAccess

type

object

patternProperties

  • [\x20-\x7E]+$

Object representation of the service to use.

properties

  • Access

#/definitions/ServicesAccess

  • ReturnValues

Set the CFN Return Value and the environment variable name you want to expose to the service

type

object

patternProperties

  • [\x20-\x7E]+$

oneOf

x-resources.common.spec.json#/definitions/varNameDef

type

object

properties

  • EnvVarName

x-resources.common.spec.json#/definitions/varNameDef

additionalProperties

False

additionalProperties

False

  • ServicesAccess

type

object

properties

  • enforceSecureConnection

Whether or not to auto-add an IAM policy that denies query not over TLS

type

boolean

default

False

  • bucket

The name of the predefined policy to use for bucket access

type

string

  • objects

The name of the predefined policy to use for objects access

type

string

Definition

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "id": "x-s3.spec.json",
  "$id": "x-s3.spec.json",
  "title": "x-s3",
  "description": "x-s3 specification for ECS Cluster",
  "type": "object",
  "properties": {
    "Lookup": {
      "$ref": "x-resources.common.spec.json#/definitions/Lookup"
    },
    "Use": {
      "type": "string"
    },
    "Properties": {
      "type": "object",
      "description": "https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-s3-bucket.html"
    },
    "Settings": {
      "$ref": "x-resources.common.spec.json#/definitions/Settings"
    },
    "MacroParameters": {
      "type": "object",
      "properties": {
        "BucketPolicy": {
          "type": "object",
          "properties": {
            "Statement": {
              "type": "array",
              "items": {
                "$ref": "services.x-iam.spec.json#/definitions/Statement"
              }
            },
            "PredefinedBucketPolicies": {
              "type": "array",
              "uniqueItems": true,
              "items": {
                "type": "string",
                "enum": [
                  "enforceSecureConnection"
                ]
              }
            }
          }
        }
      }
    },
    "Services": {
      "$ref": "#/definitions/ServicesDef"
    }
  },
  "definitions": {
    "ServicesDef": {
      "oneOf": [
        {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "name": {
                "type": "string"
              },
              "access": {
                "$ref": "#/definitions/ServicesAccess"
              }
            },
            "required": [
              "name"
            ]
          }
        },
        {
          "type": "object",
          "patternProperties": {
            "[\\x20-\\x7E]+$": {
              "description": "Object representation of the service to use.",
              "properties": {
                "Access": {
                  "$ref": "#/definitions/ServicesAccess"
                },
                "ReturnValues": {
                  "type": "object",
                  "description": "Set the CFN Return Value and the environment variable name you want to expose to the service",
                  "additionalProperties": false,
                  "patternProperties": {
                    "[\\x20-\\x7E]+$": {
                      "oneOf": [
                        {
                          "$ref": "x-resources.common.spec.json#/definitions/varNameDef"
                        },
                        {
                          "type": "object",
                          "additionalProperties": false,
                          "properties": {
                            "EnvVarName": {
                              "$ref": "x-resources.common.spec.json#/definitions/varNameDef"
                            }
                          }
                        }
                      ]
                    }
                  }
                }
              }
            }
          }
        }
      ]
    },
    "ServicesAccess": {
      "type": "object",
      "properties": {
        "enforceSecureConnection": {
          "type": "boolean",
          "description": "Whether or not to auto-add an IAM policy that denies query not over TLS",
          "default": false
        },
        "bucket": {
          "type": "string",
          "description": "The name of the predefined policy to use for bucket access"
        },
        "objects": {
          "type": "string",
          "description": "The name of the predefined policy to use for objects access"
        }
      },
      "required": [
        "bucket",
        "objects"
      ]
    }
  }
}

Test files

You can find the test files here to use as reference for your use-case.