Added support for defining docker volume configuration in ECS Params

PettitWesley · PettitWesley · commit 12832590fb3b · 2018-09-05T23:34:43.000-07:00
diff --git a/README.md b/README.md
@@ -427,6 +427,15 @@ task_definition:
       cpu_shares: integer
       mem_limit: string
       mem_reservation: string
+  docker_volumes:
+    - name: string
+      scope: string                      // Valid values: "shared" | "task"
+      autoprovision: boolean             // only valid if scope = "shared"
+      driver: string
+      driver_opts:
+        string: string
+      labels:
+        string: string
 
 run_params:
   network_configuration:
@@ -459,19 +468,7 @@ Fields listed under `task_definition` correspond to fields that will be included
   * In Docker compose version 2, the `cpu_shares`, `mem_limit`, and `mem_reservation` fields can be specified in either the compose or ECS params file. If they are specified in the ECS params file, the values will override values present in the compose file.
   * If you are using a private repository for pulling images, `repository_credentials` allows you to specify an AWS Secrets Manager secret ARN for the name of the secret containing your private repository credentials as a `credential_parameter`.
 
-Example `ecs-params.yml` with service resources specified:
-```
-version: 1
-task_definition:
-  services:
-    wordpress:
-      cpu_shares: 100
-      mem_limit: 500m
-    mysql:
-      cpu_shares: 105
-      mem_limit: 500m
-      mem_reservation: 450m
-```
+* `docker_volumes` allows you to create docker volumes. The name key is required, and `scope`, `autoprovision`, `driver`, `driver_opts` and `labels` correspond with the fields under [dockerVolumeConfiguration](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/docker-volumes.html) in an ECS Task Definition. Volumes defined with the `docker_volumes` key can be referenced in your compose file by name, even if they were not also specified in the compose file.
 
 * `task_execution_role` should be the ARN of an IAM role. **NOTE**: This field is required to enable ECS Tasks to be configured with Cloudwatch Logs, or to pull images from ECR for your tasks.
 
@@ -505,8 +502,20 @@ task_definition:
   ecs_network_mode: host
   task_role_arn: myCustomRole
   services:
-    my_service:
+    logging:
       essential: false
+    wordpress:
+      cpu_shares: 100
+      mem_limit: 500m
+    mysql:
+      cpu_shares: 105
+      mem_limit: 500m
+      mem_reservation: 450m
+  docker_volumes:
+    - name: database_volume
+      scope: shared
+      autoprovision: true
+      driver: local
 ```
 
 Example `ecs-params.yml` with network configuration with **EC2** launch type:
diff --git a/ecs-cli/modules/utils/compose/convert_task_definition.go b/ecs-cli/modules/utils/compose/convert_task_definition.go
@@ -80,10 +80,15 @@ func ConvertToTaskDefinition(taskDefinitionName string, volumes *adapter.Volumes
 		containerDefinitions = append(containerDefinitions, containerDef)
 	}
 
+	ecsVolumes, err := convertToECSVolumes(volumes, ecsParams)
+	if err != nil {
+		return nil, err
+	}
+
 	taskDefinition := &ecs.TaskDefinition{
 		Family:               aws.String(taskDefinitionName),
 		ContainerDefinitions: containerDefinitions,
-		Volumes:              convertToECSVolumes(volumes),
+		Volumes:              ecsVolumes,
 		TaskRoleArn:          aws.String(taskRoleArn),
 		NetworkMode:          aws.String(taskDefParams.networkMode),
 		Cpu:                  aws.String(taskDefParams.cpu),
@@ -291,7 +296,7 @@ func showResourceOverrideMsg(serviceName string, val int64, override int64, opti
 }
 
 // convertToECSVolumes transforms the map of hostPaths to the format of ecs.Volume
-func convertToECSVolumes(hostPaths *adapter.Volumes) []*ecs.Volume {
+func convertToECSVolumes(hostPaths *adapter.Volumes, ecsParams *ECSParams) ([]*ecs.Volume, error) {
 	output := []*ecs.Volume{}
 	// volumes with a host path
 	for hostPath, volName := range hostPaths.VolumeWithHost {
@@ -302,14 +307,54 @@ func convertToECSVolumes(hostPaths *adapter.Volumes) []*ecs.Volume {
 			}}
 		output = append(output, ecsVolume)
 	}
-	// volumes with an empty host path
-	for _, volName := range hostPaths.VolumeEmptyHost {
+
+	// volumes without host path (allowed to have Docker Volume Configuration)
+	volumesWithoutHost, err := mergeVolumesWithoutHost(hostPaths.VolumeEmptyHost, ecsParams)
+	if err != nil {
+		return nil, err
+	}
+	output = append(output, volumesWithoutHost...)
+	return output, nil
+}
+
+func mergeVolumesWithoutHost(composeVolumes []string, ecsParams *ECSParams) ([]*ecs.Volume, error) {
+	volumesWithoutHost := make(map[string]DockerVolume)
+	output := []*ecs.Volume{}
+
+	for _, volName := range composeVolumes {
+		volumesWithoutHost[volName] = DockerVolume{}
+	}
+
+	if ecsParams != nil {
+		for _, dockerVol := range ecsParams.TaskDefinition.DockerVolumes {
+			if dockerVol.Name != "" {
+				volumesWithoutHost[dockerVol.Name] = dockerVol
+			} else {
+				return nil, fmt.Errorf("Name is required when specifying a docker volume")
+			}
+		}
+	}
+
+	for volName, dVol := range volumesWithoutHost {
 		ecsVolume := &ecs.Volume{
 			Name: aws.String(volName),
 		}
+		if dVol.Name != "" {
+			ecsVolume.DockerVolumeConfiguration = &ecs.DockerVolumeConfiguration{
+				Autoprovision: dVol.Autoprovision,
+				Driver:        aws.String(dVol.Driver),
+				Scope:         aws.String(dVol.Scope),
+			}
+			if dVol.DriverOptions != nil {
+				ecsVolume.DockerVolumeConfiguration.DriverOpts = aws.StringMap(dVol.DriverOptions)
+			}
+			if dVol.Labels != nil {
+				ecsVolume.DockerVolumeConfiguration.Labels = aws.StringMap(dVol.Labels)
+			}
+		}
 		output = append(output, ecsVolume)
 	}
-	return output
+	return output, nil
 }
 
 func hasEssential(ecsParamsContainerDefs ContainerDefs, count int) bool {
diff --git a/ecs-cli/modules/utils/compose/convert_task_definition_test.go b/ecs-cli/modules/utils/compose/convert_task_definition_test.go
@@ -33,6 +33,8 @@ const (
 	containerPath2 = "/tmp/cache2"
 	hostPath       = "./cache"
 	namedVolume    = "named_volume"
+	namedVolume2   = "named_volume2"
+	namedVolume3   = "named_volume3"
 )
 
 var defaultNetwork = &yaml.Network{
@@ -1182,7 +1184,7 @@ func TestConvertToTaskDefinitionWithVolumes(t *testing.T) {
 		{
 			ContainerPath: aws.String("/tmp/cache"),
 			ReadOnly:      aws.Bool(false),
-			SourceVolume:  aws.String("volume-3"),
+			SourceVolume:  aws.String("volume-0"),
 		},
 		{
 			ContainerPath: aws.String("/tmp/cache"),
@@ -1214,6 +1216,89 @@ func TestConvertToTaskDefinitionWithVolumes(t *testing.T) {
 	assert.ElementsMatch(t, expectedVolumes, actualVolumes, "Expected volumes to match")
 }
 
+func TestConvertToTaskDefinitionWithVolumesWithHostOnly(t *testing.T) {
+	volumeConfigs := &adapter.Volumes{
+		VolumeWithHost: map[string]string{
+			hostPath: containerPath,
+		},
+	}
+
+	mountPoints := []*ecs.MountPoint{
+		{
+			ContainerPath: aws.String("/tmp/cache"),
+			ReadOnly:      aws.Bool(false),
+			SourceVolume:  aws.String("volume-0"),
+		},
+	}
+	containerConfig := adapter.ContainerConfig{
+		MountPoints: mountPoints,
+	}
+
+	containerConfigs := []adapter.ContainerConfig{containerConfig}
+
+	host := &ecs.HostVolumeProperties{SourcePath: aws.String(hostPath)}
+	expectedVolumes := []*ecs.Volume{
+		{
+			Host: host,
+			Name: aws.String(containerPath),
+		},
+	}
+
+	taskDefinition, err := ConvertToTaskDefinition(projectName, volumeConfigs, containerConfigs, "", "", nil)
+	assert.NoError(t, err, "Unexpected error converting Task Definition")
+
+	actualVolumes := taskDefinition.Volumes
+	assert.ElementsMatch(t, expectedVolumes, actualVolumes, "Expected volumes to match")
+}
+
+func TestConvertToTaskDefinitionWithECSParamsVolumeWithoutNameError(t *testing.T) {
+	volumeConfigs := &adapter.Volumes{
+		VolumeEmptyHost: []string{namedVolume, namedVolume2},
+	}
+
+	mountPoints := []*ecs.MountPoint{
+		{
+			ContainerPath: aws.String("/var/log"),
+			ReadOnly:      aws.Bool(false),
+			SourceVolume:  aws.String("named_volume"),
+		},
+		{
+			ContainerPath: aws.String("/tmp/cache"),
+			ReadOnly:      aws.Bool(false),
+			SourceVolume:  aws.String("named_volume2"),
+		},
+	}
+	containerConfig := adapter.ContainerConfig{
+		MountPoints: mountPoints,
+	}
+
+	containerConfigs := []adapter.ContainerConfig{containerConfig}
+	labels := map[string]string{
+		"testing.thisdoesntactuallyreallyadvancetheplot": "true",
+	}
+	options := map[string]string{
+		"Clyde": "says Goodbye Stranger, decides to Take The Long Way Home, and enjoys some Breakfast in America",
+		"He":    "is a big fan of 70s music",
+	}
+
+	ecsParams := &ECSParams{
+		TaskDefinition: EcsTaskDef{
+			DockerVolumes: []DockerVolume{
+				DockerVolume{
+					Autoprovision: aws.Bool(true),
+					Scope:         "shared",
+					Driver:        "local",
+					DriverOptions: options,
+					Labels:        labels,
+				},
+			},
+		},
+	}
+
+	_, err := ConvertToTaskDefinition(projectName, volumeConfigs, containerConfigs, "", "", ecsParams)
+	assert.Error(t, err, "Expected error converting Task Definition with ECS Params volume without name")
+}
+
 func TestIsZeroForEmptyConfig(t *testing.T) {
 	containerConfig := &adapter.ContainerConfig{}
 
diff --git a/ecs-cli/modules/utils/compose/ecs_params_reader.go b/ecs-cli/modules/utils/compose/ecs_params_reader.go
@@ -43,11 +43,12 @@ type ECSParams struct {
 
 // EcsTaskDef corresponds to fields in an ECS TaskDefinition
 type EcsTaskDef struct {
-	NetworkMode          string        `yaml:"ecs_network_mode"`
-	TaskRoleArn          string        `yaml:"task_role_arn"`
-	ContainerDefinitions ContainerDefs `yaml:"services"`
-	ExecutionRole        string        `yaml:"task_execution_role"`
-	TaskSize             TaskSize      `yaml:"task_size"` // Needed to run FARGATE tasks
+	NetworkMode          string         `yaml:"ecs_network_mode"`
+	TaskRoleArn          string         `yaml:"task_role_arn"`
+	ContainerDefinitions ContainerDefs  `yaml:"services"`
+	ExecutionRole        string         `yaml:"task_execution_role"`
+	TaskSize             TaskSize       `yaml:"task_size"` // Needed to run FARGATE tasks
+	DockerVolumes        []DockerVolume `yaml:"docker_volumes"`
 }
 
 // ContainerDefs is a map of ContainerDefs within a task definition
@@ -64,6 +65,15 @@ type ContainerDef struct {
 	HealthCheck       *HealthCheck           `yaml:"healthcheck"`
 }
 
+type DockerVolume struct {
+	Name          string            `yaml:"name"`
+	Scope         string            `yaml:"scope"`
+	Autoprovision *bool             `yaml:"autoprovision"`
+	Driver        string            `yaml:"driver"`
+	DriverOptions map[string]string `yaml:"driver_opts"`
+	Labels        map[string]string `yaml:"labels"`
+}
+
 // HealthCheck holds all possible fields for HealthCheck, including fields
 // supported by docker compose vs ECS
 type HealthCheck struct {
diff --git a/ecs-cli/modules/utils/compose/ecs_params_reader_test.go b/ecs-cli/modules/utils/compose/ecs_params_reader_test.go
@@ -550,6 +550,65 @@ func TestConvertToECSPlacementStrategy(t *testing.T) {
 		assert.ElementsMatch(t, expectedStrategy, ecsPlacementStrategy, "Expected placement strategy to match")
 	}
 }
+func TestReadECSParams_WithDockerVolumes(t *testing.T) {
+	ecsParamsString := `version: 1
+task_definition:
+  docker_volumes:
+    - name: my_volume
+      scope: shared
+      autoprovision: true
+      driver: doggyromcom
+      driver_opts:
+        pudding: is-engaged-to-marry-Tum-Tum
+        clyde: professes-his-love-at-the-ceremony
+        it: does-not-go-well
+        this: is-not-a-movie
+      labels:
+        pudding: mad
+        clyde: sad
+        life: sucks`
+
+	expectedVolumes := []DockerVolume{
+		DockerVolume{
+			Name:          "my_volume",
+			Scope:         "shared",
+			Autoprovision: aws.Bool(true),
+			Driver:        "doggyromcom",
+			DriverOptions: map[string]string{
+				"pudding": "is-engaged-to-marry-Tum-Tum",
+				"clyde":   "professes-his-love-at-the-ceremony",
+				"it":      "does-not-go-well",
+				"this":    "is-not-a-movie",
+			},
+			Labels: map[string]string{
+				"pudding": "mad",
+				"clyde":   "sad",
+				"life":    "sucks",
+			},
+		},
+	}
+
+	content := []byte(ecsParamsString)
+
+	tmpfile, err := ioutil.TempFile("", "ecs-params")
+	assert.NoError(t, err, "Could not create ecs-params tempfile")
+
+	ecsParamsFileName := tmpfile.Name()
+	defer os.Remove(ecsParamsFileName)
+
+	_, err = tmpfile.Write(content)
+	assert.NoError(t, err, "Could not write data to ecs-params tempfile")
+
+	err = tmpfile.Close()
+	assert.NoError(t, err, "Could not close tempfile")
+
+	ecsParams, err := ReadECSParams(ecsParamsFileName)
+
+	if assert.NoError(t, err) {
+		volumes := ecsParams.TaskDefinition.DockerVolumes
+		assert.ElementsMatch(t, expectedVolumes, volumes, "Expected volumes to match")
+	}
+}
 
 func TestReadECSParams_WithHealthCheck(t *testing.T) {
 	ecsParamsString := `version: 1
