Pre-update lifecycle hook

docs/lifecycle-hooks.md

@@ -4,6 +4,9 @@
     These are shell commands executed with `sh`, and therefore require the container to provide the `sh`
     executable.
 
+> **DO NOTE**: If the container is not running then lifecycle hooks can not run and therefore 
+> the update is executed without running any lifecycle hooks.
+
 It is possible to execute _pre/post\-check_ and _pre/post\-update_ commands
 **inside** every container updated by watchtower.
 
@@ -63,5 +66,5 @@ If the label value is explicitly set to `0`, the timeout will be disabled.
 ### Execution failure
 
 The failure of a command to execute, identified by an exit code different than
-0, will not prevent watchtower from updating the container. Only an error
+0 or 75 (EX_TEMPFAIL), will not prevent watchtower from updating the container. Only an error
 log statement containing the exit code will be reported.

internal/actions/mocks/client.go

@@ -2,6 +2,7 @@ package mocks
 
 import (
 	"errors"
+	"fmt"
 	"github.com/containrrr/watchtower/pkg/container"
 	"time"
 
@@ -70,12 +71,21 @@ func (client MockClient) RemoveImageByID(id string) error {
 
 // GetContainer is a mock method
 func (client MockClient) GetContainer(containerID string) (container.Container, error) {
-	return container.Container{}, nil
+	return client.TestData.Containers[0], nil
 }
 
 // ExecuteCommand is a mock method
-func (client MockClient) ExecuteCommand(containerID string, command string, timeout int) error {
-	return nil
+func (client MockClient) ExecuteCommand(containerID string, command string, timeout int) (SkipUpdate bool, err error) {
+	switch command {
+	case "/PreUpdateReturn0.sh":
+		return false, nil
+	case "/PreUpdateReturn1.sh":
+		return false, fmt.Errorf("command exited with code 1")
+	case "/PreUpdateReturn75.sh":
+		return true, nil
+	default:
+		return false, nil
+	}
 }
 
 // IsContainerStale is always true for the mock client

internal/actions/mocks/container.go

@@ -65,13 +65,20 @@ func CreateMockContainerWithDigest(id string, name string, image string, created
 }
 
 // CreateMockContainerWithConfig creates a container substitute valid for testing
-func CreateMockContainerWithConfig(id string, name string, image string, created time.Time, config *container2.Config) container.Container {
+func CreateMockContainerWithConfig(id string, name string, image string, running bool, restarting bool, created time.Time, config *container2.Config) container.Container {
 	content := types.ContainerJSON{
 		ContainerJSONBase: &types.ContainerJSONBase{
-			ID:      id,
-			Image:   image,
-			Name:    name,
+			ID:    id,
+			Image: image,
+			Name:  name,
+			State: &types.ContainerState{
+				Running: running,
+				Restarting: restarting,
+			},
 			Created: created.String(),
+			HostConfig: &container2.HostConfig{
+				PortBindings: map[nat.Port][]nat.PortBinding{},
+			},
 		},
 		Config: config,
 	}

internal/actions/update.go

@@ -1,6 +1,7 @@
 package actions
 
 import (
+	"errors"
 	"github.com/containrrr/watchtower/internal/util"
 	"github.com/containrrr/watchtower/pkg/container"
 	"github.com/containrrr/watchtower/pkg/lifecycle"
@@ -81,8 +82,9 @@ func Update(client container.Client, params types.UpdateParams) (*metrics2.Metri
 	if params.RollingRestart {
 		metric.Failed += performRollingRestart(containersToUpdate, client, params)
 	} else {
-		metric.Failed += stopContainersInReversedOrder(containersToUpdate, client, params)
-		metric.Failed += restartContainersInSortedOrder(containersToUpdate, client, params)
+		imageIDsOfStoppedContainers := make(map[string]bool)
+		metric.Failed, imageIDsOfStoppedContainers = stopContainersInReversedOrder(containersToUpdate, client, params)
+		metric.Failed += restartContainersInSortedOrder(containersToUpdate, client, params, imageIDsOfStoppedContainers)
 	}
 
 	metric.Updated = staleCount - (metric.Failed - staleCheckFailed)
@@ -99,13 +101,15 @@ func performRollingRestart(containers []container.Container, client container.Cl
 
 	for i := len(containers) - 1; i >= 0; i-- {
 		if containers[i].ToRestart() {
-			if err := stopStaleContainer(containers[i], client, params); err != nil {
-				failed++
-			}
-			if err := restartStaleContainer(containers[i], client, params); err != nil {
+			err := stopStaleContainer(containers[i], client, params)
+			if err != nil {
 				failed++
+			} else {
+				if err := restartStaleContainer(containers[i], client, params); err != nil {
+					failed++
+				}
+				cleanupImageIDs[containers[i].ImageID()] = true
 			}
-			cleanupImageIDs[containers[i].ImageID()] = true
 		}
 	}
 
@@ -115,14 +119,18 @@ func performRollingRestart(containers []container.Container, client container.Cl
 	return failed
 }
 
-func stopContainersInReversedOrder(containers []container.Container, client container.Client, params types.UpdateParams) int {
+func stopContainersInReversedOrder(containers []container.Container, client container.Client, params types.UpdateParams) (int, map[string]bool) {
+	imageIDsOfStoppedContainers := make(map[string]bool)
 	failed := 0
 	for i := len(containers) - 1; i >= 0; i-- {
 		if err := stopStaleContainer(containers[i], client, params); err != nil {
 			failed++
+		} else {
+			imageIDsOfStoppedContainers[containers[i].ImageID()] = true
 		}
+
 	}
-	return failed
+	return failed, imageIDsOfStoppedContainers
 }
 
 func stopStaleContainer(container container.Container, client container.Client, params types.UpdateParams) error {
@@ -135,11 +143,16 @@ func stopStaleContainer(container container.Container, client container.Client,
 		return nil
 	}
 	if params.LifecycleHooks {
-		if err := lifecycle.ExecutePreUpdateCommand(client, container); err != nil {
+		SkipUpdate, err := lifecycle.ExecutePreUpdateCommand(client, container)
+		if err != nil {
 			log.Error(err)
 			log.Info("Skipping container as the pre-update command failed")
 			return err
 		}
+		if SkipUpdate {
+			log.Debug("Skipping container as the pre-update command returned exit code 75 (EX_TEMPFAIL)")
+			return errors.New("Skipping container as the pre-update command returned exit code 75 (EX_TEMPFAIL)")
+		}
 	}
 
 	if err := client.StopContainer(container, params.Timeout); err != nil {
@@ -149,7 +162,7 @@ func stopStaleContainer(container container.Container, client container.Client,
 	return nil
 }
 
-func restartContainersInSortedOrder(containers []container.Container, client container.Client, params types.UpdateParams) int {
+func restartContainersInSortedOrder(containers []container.Container, client container.Client, params types.UpdateParams, imageIDsOfStoppedContainers map[string]bool) int {
 	imageIDs := make(map[string]bool)
 
 	failed := 0
@@ -158,10 +171,12 @@ func restartContainersInSortedOrder(containers []container.Container, client con
 		if !c.ToRestart() {
 			continue
 		}
-		if err := restartStaleContainer(c, client, params); err != nil {
-			failed++
+		if imageIDsOfStoppedContainers[c.ImageID()] {
+			if err := restartStaleContainer(c, client, params); err != nil {
+				failed++
+			}
+			imageIDs[c.ImageID()] = true
 		}
-		imageIDs[c.ImageID()] = true
 	}
 
 	if params.Cleanup {

internal/actions/update_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/containrrr/watchtower/pkg/types"
 	container2 "github.com/docker/docker/api/types/container"
 	cli "github.com/docker/docker/client"
+	"github.com/docker/go-connections/nat"
 	"time"
 
 	. "github.com/containrrr/watchtower/internal/actions/mocks"
@@ -106,6 +107,8 @@ var _ = Describe("the update action", func() {
 								"test-container-02",
 								"test-container-02",
 								"fake-image2:latest",
+								false,
+								false,
 								time.Now(),
 								&container2.Config{
 									Labels: map[string]string{
@@ -158,4 +161,187 @@ var _ = Describe("the update action", func() {
 		})
 
 	})
+
+	When("watchtower has been instructed to run lifecycle hooks", func() {
+
+		When("prupddate script returns 1", func() {
+			BeforeEach(func() {
+				client = CreateMockClient(
+					&TestData{
+						//NameOfContainerToKeep: "test-container-02",
+						Containers: []container.Container{
+							CreateMockContainerWithConfig(
+								"test-container-02",
+								"test-container-02",
+								"fake-image2:latest",
+								true,
+								false,
+								time.Now(),
+								&container2.Config{
+									Labels: map[string]string{
+										"com.centurylinklabs.watchtower.lifecycle.pre-update-timeout": "190",
+										"com.centurylinklabs.watchtower.lifecycle.pre-update":         "/PreUpdateReturn1.sh",
+									},
+									ExposedPorts: map[nat.Port]struct{}{},
+								}),
+						},
+					},
+					dockerClient,
+					false,
+					false,
+				)
+			})
+
+			It("should not update those containers", func() {
+				_, err := actions.Update(client, types.UpdateParams{Cleanup: true, LifecycleHooks: true})
+				Expect(err).NotTo(HaveOccurred())
+				Expect(client.TestData.TriedToRemoveImageCount).To(Equal(0))
+			})
+
+		})
+
+		When("prupddate script returns 75", func() {
+			BeforeEach(func() {
+				client = CreateMockClient(
+					&TestData{
+						//NameOfContainerToKeep: "test-container-02",
+						Containers: []container.Container{
+							CreateMockContainerWithConfig(
+								"test-container-02",
+								"test-container-02",
+								"fake-image2:latest",
+								true,
+								false,
+								time.Now(),
+								&container2.Config{
+									Labels: map[string]string{
+										"com.centurylinklabs.watchtower.lifecycle.pre-update-timeout": "190",
+										"com.centurylinklabs.watchtower.lifecycle.pre-update":         "/PreUpdateReturn75.sh",
+									},
+									ExposedPorts: map[nat.Port]struct{}{},
+								}),
+						},
+					},
+					dockerClient,
+					false,
+					false,
+				)
+			})
+
+			It("should not update those containers", func() {
+				_, err := actions.Update(client, types.UpdateParams{Cleanup: true, LifecycleHooks: true})
+				Expect(err).NotTo(HaveOccurred())
+				Expect(client.TestData.TriedToRemoveImageCount).To(Equal(0))
+			})
+
+		})
+
+		When("prupddate script returns 0", func() {
+			BeforeEach(func() {
+				client = CreateMockClient(
+					&TestData{
+						//NameOfContainerToKeep: "test-container-02",
+						Containers: []container.Container{
+							CreateMockContainerWithConfig(
+								"test-container-02",
+								"test-container-02",
+								"fake-image2:latest",
+								true,
+								false,
+								time.Now(),
+								&container2.Config{
+									Labels: map[string]string{
+										"com.centurylinklabs.watchtower.lifecycle.pre-update-timeout": "190",
+										"com.centurylinklabs.watchtower.lifecycle.pre-update":         "/PreUpdateReturn0.sh",
+									},
+									ExposedPorts: map[nat.Port]struct{}{},
+								}),
+						},
+					},
+					dockerClient,
+					false,
+					false,
+				)
+			})
+
+			It("should update those containers", func() {
+				_, err := actions.Update(client, types.UpdateParams{Cleanup: true, LifecycleHooks: true})
+				Expect(err).NotTo(HaveOccurred())
+				Expect(client.TestData.TriedToRemoveImageCount).To(Equal(1))
+			})
+		})
+
+		When("container is not running", func() {
+			BeforeEach(func() {
+				client = CreateMockClient(
+					&TestData{
+						//NameOfContainerToKeep: "test-container-02",
+						Containers: []container.Container{
+							CreateMockContainerWithConfig(
+								"test-container-02",
+								"test-container-02",
+								"fake-image2:latest",
+								false,
+								false,
+								time.Now(),
+								&container2.Config{
+									Labels: map[string]string{
+										"com.centurylinklabs.watchtower.lifecycle.pre-update-timeout": "190",
+										"com.centurylinklabs.watchtower.lifecycle.pre-update":         "/PreUpdateReturn1.sh",
+									},
+									ExposedPorts: map[nat.Port]struct{}{},
+								}),
+						},
+					},
+					dockerClient,
+					false,
+					false,
+				)
+			})
+
+			It("skip running preupdate", func() {
+				_, err := actions.Update(client, types.UpdateParams{Cleanup: true, LifecycleHooks: true})
+				Expect(err).NotTo(HaveOccurred())
+				Expect(client.TestData.TriedToRemoveImageCount).To(Equal(1))
+			})
+
+		})
+
+		When("container is restarting", func() {
+			BeforeEach(func() {
+				client = CreateMockClient(
+					&TestData{
+						//NameOfContainerToKeep: "test-container-02",
+						Containers: []container.Container{
+							CreateMockContainerWithConfig(
+								"test-container-02",
+								"test-container-02",
+								"fake-image2:latest",
+								false,
+								true,
+								time.Now(),
+								&container2.Config{
+									Labels: map[string]string{
+										"com.centurylinklabs.watchtower.lifecycle.pre-update-timeout": "190",
+										"com.centurylinklabs.watchtower.lifecycle.pre-update":         "/PreUpdateReturn1.sh",
+									},
+									ExposedPorts: map[nat.Port]struct{}{},
+								}),
+						},
+					},
+					dockerClient,
+					false,
+					false,
+				)
+			})
+
+			It("skip running preupdate", func() {
+				_, err := actions.Update(client, types.UpdateParams{Cleanup: true, LifecycleHooks: true})
+				Expect(err).NotTo(HaveOccurred())
+				Expect(client.TestData.TriedToRemoveImageCount).To(Equal(1))
+			})
+
+		})
+
+	})
 })