-
Notifications
You must be signed in to change notification settings - Fork 18
chore(go): add performance benchmark #1983
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 37 commits
48774e2
5bc7702
a604509
7c16d49
d682110
8f3330c
286ec92
7b7da98
321cbed
9d5e62c
02f848e
2d62147
fb3967d
23f15e4
8aa8bf2
bbbf5ae
cc57cbe
b238461
9e7402a
fbca12a
f608db0
317d803
ca56d0d
b22f0c8
4bf0bd8
6c93e74
3fc071a
e9e5c73
69800a9
2e4d8f6
f0e068f
642a1ca
4eafece
c829ef0
d466b5c
fe73ce6
2dd9e98
8b1dc27
0c2a4f7
f0938ee
fa8a1b4
e8c6194
0615ec9
729bd37
684f8e4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| # This workflow runs every day 09:00 UTC (1AM PST) | ||
| name: Performance Benchmarks | ||
|
|
||
| on: | ||
| pull_request: | ||
| paths: | ||
| - ".github/workflows/run-benchmark.yml" | ||
| workflow_call: | ||
| inputs: | ||
| dafny: | ||
| description: "The Dafny version to run" | ||
| required: false | ||
| default: "4.9.0" | ||
| type: string | ||
| regenerate-code: | ||
| description: "Regenerate code using smithy-dafny" | ||
| required: false | ||
| default: false | ||
| type: boolean | ||
| mpl-version: | ||
| description: "MPL version to use" | ||
| required: false | ||
| type: string | ||
| mpl-head: | ||
| description: "Running on MPL HEAD" | ||
| required: false | ||
| default: false | ||
| type: boolean | ||
| jobs: | ||
| testGo: | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| library: [DynamoDbEncryption] | ||
| benchmark-dir: [db-esdk-performance-testing] | ||
| os: [ubuntu-22.04, macos-15-intel] | ||
| go-version: ["1.23", "1.24", "1.25"] | ||
| runs-on: ${{ matrix.os }} | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/configure-aws-credentials@v5 | ||
| with: | ||
| aws-region: us-west-2 | ||
| role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-DDBEC-Dafny-Role-us-west-2 | ||
| role-session-name: DDBEC-Performance-Benchmarks-Go | ||
|
|
||
| - name: Support longpaths | ||
| run: | | ||
| git config --global core.longpaths true | ||
|
|
||
| - uses: actions/checkout@v5 | ||
| with: | ||
| submodules: recursive | ||
| - name: Init Submodules | ||
| shell: bash | ||
| run: | | ||
| git submodule update --init --recursive submodules/smithy-dafny | ||
| git submodule update --init --recursive submodules/MaterialProviders | ||
|
|
||
| - name: Create temporary global.json | ||
| run: echo '{"sdk":{"rollForward":"latestFeature","version":"6.0.0"}}' > ./global.json | ||
|
|
||
| - name: Setup Java 17 for codegen | ||
| uses: actions/setup-java@v5 | ||
| with: | ||
| distribution: "corretto" | ||
| java-version: "17" | ||
|
|
||
| - name: Update MPL submodule if using MPL HEAD | ||
| if: ${{ inputs.mpl-head == true }} | ||
| working-directory: submodules/MaterialProviders | ||
| run: | | ||
| git checkout main | ||
| git pull | ||
| git submodule update --init --recursive | ||
| git rev-parse HEAD | ||
|
|
||
| - name: Update project.properties if using MPL HEAD | ||
| if: ${{ inputs.mpl-head == true }} | ||
| run: | | ||
| sed "s/mplDependencyJavaVersion=.*/mplDependencyJavaVersion=${{inputs.mpl-version}}/g" project.properties > project.properties2; mv project.properties2 project.properties | ||
|
|
||
| - name: Install Go | ||
| uses: actions/setup-go@v6 | ||
| with: | ||
| go-version: ${{ matrix.go-version }} | ||
|
|
||
| - name: Setup Dafny | ||
| uses: dafny-lang/setup-dafny-action@v1.8.0 | ||
| with: | ||
| dafny-version: "4.9.0" | ||
|
|
||
| - name: Install Smithy-Dafny codegen dependencies | ||
| uses: ./.github/actions/install_smithy_dafny_codegen_dependencies | ||
|
|
||
| - name: Build ${{ matrix.library }} implementation | ||
| shell: bash | ||
| working-directory: ./${{ matrix.library }} | ||
| run: | | ||
| # This works because `node` is installed by default on GHA runners | ||
| CORES=$(node -e 'console.log(os.cpus().length)') | ||
| make transpile_go CORES=$CORES | ||
|
|
||
| - name: Run Performance Benchmarks - Quick Mode | ||
| shell: bash | ||
| working-directory: ./${{matrix.benchmark-dir}}/benchmarks/go | ||
| run: | | ||
| go run . --config ../config/test-scenarios.yaml --quick | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| # DB-ESDK Performance Test Scenarios Configuration | ||
|
|
||
| # Data sizes to test (in bytes) | ||
| # Categories are for organization only - code processes all sizes regardless of category | ||
| data_sizes: | ||
| small: | ||
| - 1024 # 1KB | ||
| - 5120 # 5KB | ||
| - 10240 # 10KB | ||
| medium: | ||
| - 102400 # 100KB | ||
| - 512000 # 500KB | ||
| - 1048576 # 1MB | ||
| - 10000000 | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this was the conversation about MegaBytes and MebiBytes - In any case, aws refers MB as 2^10 so we can drop the 10^6 values (would reduce suite runtime).
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed it here: 729bd37 |
||
| large: | ||
| - 10485760 # 10MB | ||
| - 52428800 # 50MB | ||
| - 104857600 | ||
| - 100000000 # 100MB | ||
|
|
||
| # Quick test configuration (reduced test set for faster execution) | ||
| quick_config: | ||
| data_sizes: | ||
| small: | ||
| - 102400 # 100KB - within DynamoDB's 400KB limit | ||
| iterations: | ||
| warmup: 3 # Reduced warmup iterations | ||
| measurement: 3 # Reduced measurement iterations | ||
| concurrency_levels: | ||
| - 1 | ||
| - 2 | ||
| test_types: | ||
| - "throughput" | ||
| - "memory" | ||
| - "concurrency" | ||
|
|
||
| # Test iterations for statistical significance | ||
| iterations: | ||
| warmup: 5 # Warmup iterations (not counted) | ||
| measurement: 10 # Measurement iterations | ||
|
|
||
| # Concurrency levels to test | ||
| concurrency_levels: | ||
|
ShubhamChaturvedi7 marked this conversation as resolved.
|
||
| # - 1 | ||
| # - 2 | ||
| # - 4 | ||
| # - 8 | ||
| # - 16 | ||
|
|
||
| # DynamoDB table name | ||
| table_name: "dbesdk-performance-testing" | ||
|
|
||
| # Keyring | ||
| keyring: "raw-aes" | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| # ESDK Go Benchmark | ||
|
|
||
| Performance benchmark suite for the AWS Encryption SDK (ESDK) Go implementation. | ||
|
|
||
| ## Quick Start | ||
|
|
||
| ```bash | ||
| # Run quick benchmark | ||
| go run . --config ../../config/test-scenarios.yaml --quick | ||
|
|
||
| # Run full benchmark | ||
| go run . --config ../../config/test-scenarios.yaml | ||
| ``` | ||
|
|
||
| ## Build | ||
|
|
||
| ```bash | ||
| # Build release binary | ||
| go build -o esdk-benchmark . | ||
|
|
||
| # Run built binary | ||
| ./esdk-benchmark --quick | ||
| ``` | ||
|
|
||
| ## Configuration | ||
|
|
||
| The benchmark uses YAML configuration files. See `../../config/test-scenarios.yaml` for the full configuration format. | ||
|
|
||
| ### Quick Mode | ||
|
|
||
| Quick mode runs a subset of tests with reduced iterations: | ||
|
|
||
| - Only runs test types specified in `quick_config.test_types` | ||
| - Uses smaller data sizes from `quick_config.data_sizes.small` | ||
| - Fewer iterations: `quick_config.iterations.measurement` | ||
|
|
||
| ## Test Types | ||
|
|
||
| - **throughput**: Measures operations per second and latency | ||
| - **memory**: Measures peak memory usage during operations | ||
| - **concurrency**: Tests performance under concurrent load | ||
|
|
||
| ## Output | ||
|
|
||
| Results are saved to JSON format in `../../results/raw-data/go_results.json` by default. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package benchmark | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "reflect" | ||
| "runtime/metrics" | ||
| "time" | ||
|
|
||
| dbesdkitemencryptortypes "github.com/aws/aws-database-encryption-sdk-dynamodb/releases/go/dynamodb-esdk/awscryptographydbencryptionsdkdynamodbitemencryptorsmithygeneratedtypes" | ||
| "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" | ||
| ) | ||
|
|
||
| // === Helper Functions === | ||
|
|
||
| // runItemEncryptorCycle performs a item encryptor encrypt and decrypt cycle with 25 items and measures performance | ||
|
ShubhamChaturvedi7 marked this conversation as resolved.
Outdated
|
||
| func (b *DBESDKBenchmark) runItemEncryptorCycle(data []byte) (float64, float64, error) { | ||
| item := map[string]types.AttributeValue{ | ||
| "partition_key": &types.AttributeValueMemberS{Value: "benchmark-test"}, | ||
| "sort_key": &types.AttributeValueMemberN{Value: "0"}, | ||
| "attribute1": &types.AttributeValueMemberM{Value: map[string]types.AttributeValue{ | ||
| "data": &types.AttributeValueMemberB{Value: data}, | ||
| }}, | ||
| "attribute2": &types.AttributeValueMemberS{Value: "sign me!"}, | ||
| ":attribute3": &types.AttributeValueMemberS{Value: "ignore me!"}, | ||
| } | ||
|
|
||
| encryptItemInput := &dbesdkitemencryptortypes.EncryptItemInput{ | ||
| PlaintextItem: item, | ||
| } | ||
|
|
||
| encryptItemStart := time.Now() | ||
| encryptItemOutput, err := b.ItemEncryptorClient.EncryptItem(context.Background(), *encryptItemInput) | ||
| if err != nil { | ||
| return 0, 0, fmt.Errorf("EncryptItem failed: %w", err) | ||
| } | ||
| encryptItemDuration := time.Since(encryptItemStart).Seconds() * 1000 | ||
| // Demonstrate that the item has been encrypted | ||
| encryptedItem := encryptItemOutput.EncryptedItem | ||
|
|
||
| // Directly decrypt the encrypted item using the DynamoDb Item Encryptor | ||
| decryptItemInput := &dbesdkitemencryptortypes.DecryptItemInput{ | ||
| EncryptedItem: encryptedItem, | ||
| } | ||
| decryptItemStart := time.Now() | ||
| decryptedItem, err := b.ItemEncryptorClient.DecryptItem(context.Background(), *decryptItemInput) | ||
| if err != nil { | ||
| return 0, 0, fmt.Errorf("EncryptItem failed: %w", err) | ||
| } | ||
| decryptItemDuration := time.Since(decryptItemStart).Seconds() * 1000 | ||
|
|
||
| if !reflect.DeepEqual(item, decryptedItem.PlaintextItem) { | ||
| panic("Decrypted item does not match original item") | ||
| } | ||
| return encryptItemDuration, decryptItemDuration, nil | ||
| } | ||
|
|
||
| // shouldRunTestType checks if a test type should be run based on quick config | ||
| func (b *DBESDKBenchmark) shouldRunTestType(testType string) bool { | ||
| if b.Config.QuickConfig == nil || len(b.Config.QuickConfig.TestTypes) == 0 { | ||
| return true | ||
| } | ||
|
|
||
| for _, allowedType := range b.Config.QuickConfig.TestTypes { | ||
| if allowedType == testType { | ||
| return true | ||
| } | ||
| } | ||
| return false | ||
| } | ||
|
|
||
| // === Memory Test Implementation === | ||
|
|
||
| // sampleMemoryContinuously runs continuous memory sampling during operation | ||
| func (b *DBESDKBenchmark) sampleMemoryContinuously(beforeHeap, beforeAllocs uint64, stopChan chan bool) []MemorySample { | ||
| var samples []MemorySample | ||
| ticker := time.NewTicker(SamplingIntervalMs * time.Millisecond) | ||
| defer ticker.Stop() | ||
|
|
||
| for { | ||
| select { | ||
| case <-stopChan: | ||
| return samples | ||
| case <-ticker.C: | ||
| var currentSamples [2]metrics.Sample | ||
| currentSamples[0].Name = "/memory/classes/heap/objects:bytes" | ||
| currentSamples[1].Name = "/gc/heap/allocs:bytes" | ||
| metrics.Read(currentSamples[:]) | ||
|
|
||
| var heapDelta, allocsDelta uint64 | ||
| if currentSamples[0].Value.Uint64() > beforeHeap { | ||
| heapDelta = currentSamples[0].Value.Uint64() - beforeHeap | ||
| } | ||
| if currentSamples[1].Value.Uint64() > beforeAllocs { | ||
| allocsDelta = currentSamples[1].Value.Uint64() - beforeAllocs | ||
| } | ||
|
|
||
| sample := MemorySample{ | ||
| Timestamp: time.Now(), | ||
| HeapMB: float64(heapDelta) / (1024 * 1024), | ||
| MetricsAllocsMB: float64(allocsDelta) / (1024 * 1024), | ||
| MemStatsAllocsMB: 0, | ||
| } | ||
| samples = append(samples, sample) | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package benchmark | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
|
|
||
| "gopkg.in/yaml.v3" | ||
| ) | ||
|
|
||
| type KeyringType string | ||
|
|
||
| const ( | ||
| RawAESKeying KeyringType = "raw-aes" | ||
| ) | ||
|
|
||
| // TestConfig represents the configuration for benchmark tests | ||
| type TestConfig struct { | ||
| DataSizes struct { | ||
| Small []int `yaml:"small"` | ||
| Medium []int `yaml:"medium"` | ||
| Large []int `yaml:"large"` | ||
| } `yaml:"data_sizes"` | ||
| Iterations struct { | ||
| Warmup int `yaml:"warmup"` | ||
| Measurement int `yaml:"measurement"` | ||
| } `yaml:"iterations"` | ||
| ConcurrencyLevels []int `yaml:"concurrency_levels"` | ||
| QuickConfig *QuickConfig `yaml:"quick_config"` | ||
| TableName string `yaml:"table_name"` | ||
| Keyring KeyringType `yaml:"keyring"` | ||
| } | ||
|
|
||
| // QuickConfig represents the quick test configuration | ||
| type QuickConfig struct { | ||
| DataSizes struct { | ||
| Small []int `yaml:"small"` | ||
| } `yaml:"data_sizes"` | ||
| Iterations struct { | ||
| Warmup int `yaml:"warmup"` | ||
| Measurement int `yaml:"measurement"` | ||
| } `yaml:"iterations"` | ||
| ConcurrencyLevels []int `yaml:"concurrency_levels"` | ||
| TestTypes []string `yaml:"test_types"` | ||
| } | ||
|
|
||
| // LoadConfig loads the test configuration from YAML file | ||
| func LoadConfig(configPath string) (TestConfig, error) { | ||
| var config TestConfig | ||
|
|
||
| if _, err := os.Stat(configPath); os.IsNotExist(err) { | ||
| return config, fmt.Errorf("config file not found: %s", configPath) | ||
| } | ||
|
|
||
| data, err := os.ReadFile(configPath) | ||
| if err != nil { | ||
| return config, fmt.Errorf("failed to read config file: %w", err) | ||
| } | ||
|
|
||
| if err := yaml.Unmarshal(data, &config); err != nil { | ||
| return config, fmt.Errorf("failed to parse config file: %w", err) | ||
| } | ||
|
|
||
| return config, nil | ||
| } |
Uh oh!
There was an error while loading. Please reload this page.