Overhaul workflow verification

.github/workflows/docker-images.yml

@@ -82,11 +82,19 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PAT }}
 
+      - name: Extract branch name
+        id: extract_branch
+        run: |
+          BRANCH_NAME="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+          SANITIZED_BRANCH=$(echo "$BRANCH_NAME" | sed 's/\//-/g' | sed 's/[^a-zA-Z0-9._-]/-/g')
+          echo "branch_name=$SANITIZED_BRANCH" >> $GITHUB_OUTPUT
+
       - name: Build and push to Docker Hub
         env:
           LANG: ${{ inputs.lang }}
           SDK_VERSION: ${{ inputs.sdk-version || 'checked-out-sdk/' }}
-          IMAGE_TAG_ARGS: ${{ inputs.sdk-repo-ref && format('--image-tag {0}-{1}', inputs.lang, inputs.docker-tag-ext) || ''}}
+          BRANCH_TAG_COMPONENT: ${{ inputs.lang && format('{0}-{1}', inputs.lang, steps.extract_branch.outputs.branch_name) || format('cli-{0}', steps.extract_branch.outputs.branch_name) }}
+          IMAGE_TAG_ARGS: ${{ inputs.sdk-repo-ref && format('--image-tag {0}-{1} --image-tag {2}', inputs.lang, inputs.docker-tag-ext, inputs.lang && format('{0}-{1}', inputs.lang, steps.extract_branch.outputs.branch_name) || format('cli-{0}', steps.extract_branch.outputs.branch_name)) || format('--image-tag {0}', inputs.lang && format('{0}-{1}', inputs.lang, steps.extract_branch.outputs.branch_name) || format('cli-{0}', steps.extract_branch.outputs.branch_name)) }}
           TAG_LATEST_ARGS: ${{ inputs.as-latest && '--tag-as-latest' || ''}}
           LANG_ARGS: ${{ inputs.lang && format('--language {0}', inputs.lang) || '' }}
           VERSION_ARGS: ${{ inputs.sdk-version && format('--version {0}', inputs.sdk-version) || '' }}

cmd/cli/run_scenario.go

@@ -2,11 +2,15 @@ package cli
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
 	"fmt"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/antithesishq/antithesis-sdk-go/assert"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/temporalio/omes/cmd/clioptions"
@@ -56,7 +60,7 @@ type scenarioRunConfig struct {
 	scenarioOptions               []string
 	timeout                       time.Duration
 	doNotRegisterSearchAttributes bool
-	ignoreAlreadyStarted          bool
+	verificationTimeout           time.Duration
 }
 
 func (r *scenarioRunner) addCLIFlags(fs *pflag.FlagSet) {
@@ -82,8 +86,8 @@ func (r *scenarioRunConfig) addCLIFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&r.doNotRegisterSearchAttributes, "do-not-register-search-attributes", false,
 		"Do not register the default search attributes used by scenarios. "+
 			"If the search attributes are not registed by the scenario they must be registered through some other method")
-	fs.BoolVar(&r.ignoreAlreadyStarted, "ignore-already-started", false,
-		"Ignore if a workflow with the same ID already exists. A Scenario may choose to override this behavior.")
+	fs.DurationVar(&r.verificationTimeout, "verification-timeout", 2*time.Minute,
+		"Maximum duration to wait for post-scenario verification (default 2m).")
 }
 
 func (r *scenarioRunner) preRun() {
@@ -98,6 +102,8 @@ func (r *scenarioRunner) run(ctx context.Context) error {
 		return fmt.Errorf("run ID not found")
 	} else if r.iterations > 0 && r.duration > 0 {
 		return fmt.Errorf("cannot provide both iterations and duration")
+	} else if r.verificationTimeout <= 0 {
+		return fmt.Errorf("verification-timeout must be greater than 0")
 	}
 
 	// Parse options
@@ -137,6 +143,8 @@ func (r *scenarioRunner) run(ctx context.Context) error {
 		}
 		// Wait 300ms and try again
 		time.Sleep(300 * time.Millisecond)
+
+		r.logger.Error("Failed to dial, retrying ...", zap.Error(err))
 	}
 	defer client.Close()
 
@@ -145,9 +153,16 @@ func (r *scenarioRunner) run(ctx context.Context) error {
 		return fmt.Errorf("failed to get root directory: %w", err)
 	}
 
+	// Generate a random execution ID to ensure no two executions with the same RunID collide
+	executionID, err := generateExecutionID()
+	if err != nil {
+		return fmt.Errorf("failed to generate execution ID: %w", err)
+	}
+
 	scenarioInfo := loadgen.ScenarioInfo{
 		ScenarioName:   r.scenario.Scenario,
 		RunID:          r.scenario.RunID,
+		ExecutionID:    executionID,
 		Logger:         r.logger,
 		MetricsHandler: metrics.NewHandler(),
 		Client:         client,
@@ -159,16 +174,45 @@ func (r *scenarioRunner) run(ctx context.Context) error {
 			MaxIterationAttempts:          r.maxIterationAttempts,
 			Timeout:                       r.timeout,
 			DoNotRegisterSearchAttributes: r.doNotRegisterSearchAttributes,
-			IgnoreAlreadyStarted:          r.ignoreAlreadyStarted,
 		},
 		ScenarioOptions: scenarioOptions,
 		Namespace:       r.clientOptions.Namespace,
 		RootPath:        repoDir,
 	}
 	executor := scenario.ExecutorFn()
-	err = executor.Run(ctx, scenarioInfo)
-	if err != nil {
-		return fmt.Errorf("failed scenario: %w", err)
+
+	// 1. Run the scenario
+	scenarioErr := executor.Run(ctx, scenarioInfo)
+
+	// Collect all errors
+	var allErrors []error
+	if scenarioErr != nil {
+		allErrors = append(allErrors, fmt.Errorf("scenario execution failed: %w", scenarioErr))
+		assert.Unreachable("scenario execution failed", map[string]any{"error": scenarioErr})
+	}
+
+	verifyCtx, verifyCancel := context.WithTimeout(ctx, r.verificationTimeout)
+	defer verifyCancel()
+
+	// 2. Run verifications
+	if scenario.VerifyFn != nil {
+		verifyErrs := scenario.VerifyFn(verifyCtx, scenarioInfo, executor)
+		for _, err := range verifyErrs {
+			allErrors = append(allErrors, fmt.Errorf("post-scenario verification failed: %w", err))
+			assert.Unreachable("post-scenario verification failed", map[string]any{"error": err})
+		}
+	}
+
+	// Aggregate all errors
+	return errors.Join(allErrors...)
+}
+
+// generateExecutionID generates a random execution ID to uniquely identify this particular
+// execution of a scenario. This ensures no two executions with the same RunID collide.
+func generateExecutionID() (string, error) {
+	bytes := make([]byte, 8) // 8 bytes = 16 hex characters
+	if _, err := rand.Read(bytes); err != nil {
+		return "", err
 	}
-	return nil
+	return hex.EncodeToString(bytes), nil
 }

cmd/cli/run_scenario_with_worker.go

@@ -80,6 +80,7 @@ func (r *workerWithScenarioRunner) run(ctx context.Context) error {
 			maxIterationsPerSecond:        r.maxIterationsPerSecond,
 			scenarioOptions:               r.scenarioOptions,
 			timeout:                       r.timeout,
+			verificationTimeout:           r.verificationTimeout,
 			doNotRegisterSearchAttributes: r.doNotRegisterSearchAttributes,
 		},
 		clientOptions:  r.ClientOptions,

dockerfiles/cli.Dockerfile

@@ -1,61 +1,101 @@
 # Build in a full featured container
 ARG TARGETARCH
 
-FROM --platform=linux/$TARGETARCH golang:1.25 AS build
+# Source stage: prepare source code and install Antithesis SDK
+FROM --platform=linux/$TARGETARCH golang:1.25 AS source
 
 WORKDIR /app
 
 # Install protobuf compiler and git
 RUN apt-get update \
-  && DEBIAN_FRONTEND=noninteractive \
+    && DEBIAN_FRONTEND=noninteractive \
     apt-get install --no-install-recommends --assume-yes \
-      protobuf-compiler=3.21.12-11 libprotoc-dev=3.21.12-11 \
-  && rm -rf /var/lib/apt/lists/*
+    protobuf-compiler=3.21.12-11 libprotoc-dev=3.21.12-11 \
+    && rm -rf /var/lib/apt/lists/*
 
-# Install Rust for kitchen-sink-gen
-RUN wget -q -O - https://sh.rustup.rs | sh -s -- -y \
-  && . $HOME/.cargo/env \
-  && echo "TARGETARCH: $TARGETARCH" \
-  && ARCH=$(uname -m) \
-  && echo "uname -m: $ARCH" \
-  && if [ "$TARGETARCH" = "arm64" ] || [ "$ARCH" = "aarch64" ]; then \
-       rustup target add aarch64-unknown-linux-musl; \
-     else \
-       rustup target add x86_64-unknown-linux-musl; \
-     fi
-ENV PATH="$PATH:/root/.cargo/bin"
-
-# Copy CLI build dependencies
+# Copy all source code
 COPY cmd ./cmd
 COPY loadgen ./loadgen
 COPY scenarios ./scenarios
 COPY workers ./workers/
 COPY go.mod go.sum ./
 
+# Install Antithesis SDK and instrumentor
+RUN go get github.com/antithesishq/antithesis-sdk-go@feature-assertion-wrappers && \
+    go install github.com/antithesishq/antithesis-sdk-go/tools/antithesis-go-instrumentor@feature-assertion-wrappers
+
+# Instrumented stage: instrument the code with Antithesis
+FROM --platform=linux/$TARGETARCH golang:1.25 AS instrumented
+
+# Copy source and instrumentor
+COPY --from=source /app /app
+COPY --from=source /go/bin/antithesis-go-instrumentor /go/bin/antithesis-go-instrumentor
+COPY --from=source /go/pkg/mod /go/pkg/mod
+
+WORKDIR /app
+
+RUN mkdir /app_transformed && \
+    antithesis-go-instrumentor /app /app_transformed
+
+# Build stage: compile the instrumented code
+FROM --platform=linux/$TARGETARCH golang:1.25 AS build
+
+ARG TARGETARCH
+
+# Install protobuf compiler and git
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive \
+    apt-get install --no-install-recommends --assume-yes \
+    protobuf-compiler=3.21.12-11 libprotoc-dev=3.21.12-11 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Rust for kitchen-sink-gen
+RUN wget -q -O - https://sh.rustup.rs | sh -s -- -y \
+    && . $HOME/.cargo/env \
+    && echo "TARGETARCH: $TARGETARCH" \
+    && ARCH=$(uname -m) \
+    && echo "uname -m: $ARCH" \
+    && if [ "$TARGETARCH" = "arm64" ] || [ "$ARCH" = "aarch64" ]; then \
+    rustup target add aarch64-unknown-linux-musl; \
+    else \
+    rustup target add x86_64-unknown-linux-musl; \
+    fi
+ENV PATH="$PATH:/root/.cargo/bin"
+
+# Copy entire instrumented structure
+COPY --from=instrumented /app_transformed /app_transformed
+
+# Set working directory to the customer code
+WORKDIR /app_transformed/customer
+
 # Build the CLI
-RUN CGO_ENABLED=0 go build -o temporal-omes ./cmd
+RUN CGO_ENABLED=0 go build -o temporal-omes -tags with_antithesis_sdk ./cmd
 
 # Install protoc-gen-go for kitchen-sink-gen build
 RUN go install google.golang.org/protobuf/cmd/[email protected]
 
 # Build kitchen-sink-gen (statically linked)
 RUN cd loadgen/kitchen-sink-gen && \
-  echo "TARGETARCH: $TARGETARCH" && \
-  ARCH=$(uname -m) && \
-  echo "uname -m: $ARCH" && \
-  if [ "$TARGETARCH" = "arm64" ] || [ "$ARCH" = "aarch64" ]; then \
+    echo "TARGETARCH: $TARGETARCH" && \
+    ARCH=$(uname -m) && \
+    echo "uname -m: $ARCH" && \
+    if [ "$TARGETARCH" = "arm64" ] || [ "$ARCH" = "aarch64" ]; then \
     RUST_TARGET=aarch64-unknown-linux-musl; \
-  else \
+    else \
     RUST_TARGET=x86_64-unknown-linux-musl; \
-  fi && \
-  echo "Building for rust target: $RUST_TARGET" && \
-  RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target $RUST_TARGET
+    fi && \
+    echo "Building for rust target: $RUST_TARGET" && \
+    RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target $RUST_TARGET
 
 # Copy the CLI to a distroless "run" container
 FROM --platform=linux/$TARGETARCH gcr.io/distroless/static-debian11:nonroot
 
-COPY --from=build /app/temporal-omes /app/temporal-omes
-COPY --from=build /app/loadgen/kitchen-sink-gen/target/*/release/kitchen-sink-gen /app/kitchen-sink-gen
+COPY --from=build /app_transformed/customer/temporal-omes /app/temporal-omes
+COPY --from=build /app_transformed/customer/loadgen/kitchen-sink-gen/target/*/release/kitchen-sink-gen /app/kitchen-sink-gen
+
+# Copy instrumentation metadata
+COPY --from=instrumented /app_transformed/notifier /notifier
+COPY --from=instrumented /app_transformed/symbols /symbols
 
 # Default entrypoint for CLI usage
-ENTRYPOINT ["/app/temporal-omes"]
+ENTRYPOINT ["/app/temporal-omes"]

go.mod

@@ -3,6 +3,7 @@ module github.com/temporalio/omes
 go 1.25.0
 
 require (
+	github.com/antithesishq/antithesis-sdk-go v0.5.1-0.20250924165633-f60b0222f1b6
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/protobuf v1.5.4
 	github.com/google/uuid v1.6.0

go.sum

@@ -1,3 +1,5 @@
+github.com/antithesishq/antithesis-sdk-go v0.5.1-0.20250924165633-f60b0222f1b6 h1:qSD74Vz3scN2SrfML8dy2Whcv0C3pNkfqYZXeL4SIq0=
+github.com/antithesishq/antithesis-sdk-go v0.5.1-0.20250924165633-f60b0222f1b6/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=