Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
d1c9b77
feat(benchmarks): add standalone blobstore benchmark harness
serdarozerr Nov 5, 2026
840b490
feat: html and csv reports are removed
serdarozerr Nov 5, 2026
9226d19
feat: workflow for manually benchmarking is added
serdarozerr Nov 6, 2026
b92e7b1
feat: temproraryl added push to test workflow
serdarozerr Nov 6, 2026
ae7ac70
fix: fix path
serdarozerr Nov 6, 2026
d7f7ee1
feat: add default provider for testing
serdarozerr Nov 6, 2026
d6c9b37
fix: reexport the aws credential to use util funtions
serdarozerr Nov 6, 2026
aeb1e15
fix: try gcs quick with default values
serdarozerr Nov 6, 2026
1353361
fix: try gcs quick profile
serdarozerr Nov 6, 2026
9f61de4
fix: try azurebs quick profile
serdarozerr Nov 6, 2026
85cd2fb
fix: try alioss quick profile
serdarozerr Nov 6, 2026
df790e1
fix: try alioss quick profile
serdarozerr Nov 6, 2026
8da8992
fix: benchmarking tests are added into release cron
serdarozerr Nov 6, 2026
3b73f12
fix: temporary conde changes are reverted
serdarozerr Nov 6, 2026
513fbdb
fix: use version in benchmark tests
serdarozerr Nov 6, 2026
41e4407
fix: remove gcs json on exit
serdarozerr Nov 9, 2026
29890c5
fix: add benchmark tag to linter config
serdarozerr Nov 9, 2026
b51c4d0
feat: bechmark module's unit tests are addeed into workflow
serdarozerr Nov 9, 2026
2d852ca
test: remove unrelated test
serdarozerr Nov 10, 2026
3d6523a
fix: README.md updated
serdarozerr Nov 10, 2026
00aef25
fix: Fail calculation is refactored
serdarozerr Nov 10, 2026
16b93f1
fix: golint error fixed
serdarozerr Nov 10, 2026
0526e10
fix: remove delete request when put failed
serdarozerr Nov 10, 2026
ffd8911
fix: step summary for failed benchmar tests are fixed
serdarozerr Nov 10, 2026
cbc7aa5
fix: runErr details from subprocess are honored
serdarozerr Nov 11, 2026
b007730
fix: created cloudformation config for benchmark tests for s3
serdarozerr Nov 11, 2026
8c4d4bd
fix: default params are reverted
serdarozerr Nov 11, 2026
ffb261b
feat: stack name generated from github run-id to not collide when s3 …
serdarozerr Nov 11, 2026
31cc4b0
fix: default params are reverted
serdarozerr Nov 11, 2026
1ada4ac
fix: s3 iam integration tests refactored.
serdarozerr Nov 13, 2026
e2070d4
fix: add line break
serdarozerr Nov 13, 2026
0d66389
fix: for sucessfull but integrity failed runs still report the time. …
serdarozerr Nov 13, 2026
a124ebb
fix: remove temporary test iam workflow
serdarozerr Nov 13, 2026
cc09f89
Merge branch 'main' into feature/benchmark
stephanme Nov 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 140 additions & 0 deletions .github/scripts/benchmark/setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
#!/usr/bin/env bash
set -euo pipefail

script_dir="$( cd "$(dirname "${0}")" && pwd )"


: "${benchmark_config_file:?}"
: "${provider:?provider is required (s3|gcs|azurebs|alioss)}"
CONFIG_FILE="${benchmark_config_file}"

case "${provider}" in

s3)
: "${aws_access_key_id:?}"
: "${aws_secret_access_key:?}"
: "${aws_region:=us-east-1}"
: "${aws_stack_name:=s3cli-benchmark-${GITHUB_RUN_ID:-local}}"

# translate to names the s3 setup-aws-infrastructure.sh script expects
export access_key_id="${aws_access_key_id}"
export secret_access_key="${aws_secret_access_key}"
export region_name="${aws_region}"
export stack_name="${aws_stack_name}"
export template_name="s3cli-benchmark"

bash "${script_dir}/../s3/setup-aws-infrastructure.sh"

# source utils to use get_stack_info functions
# re-export AWS vars since subprocess env does not propagate back
export AWS_ACCESS_KEY_ID="${aws_access_key_id}"
export AWS_SECRET_ACCESS_KEY="${aws_secret_access_key}"
export AWS_DEFAULT_REGION="${aws_region}"
source "${script_dir}/../s3/utils.sh"
stack_info=$(get_stack_info "${aws_stack_name}")
bucket_name=$(get_stack_info_of "${stack_info}" "BucketName")
: "${bucket_name:?failed to read bucket name from stack outputs}"

jq -n \
--arg access_key_id "${aws_access_key_id}" \
--arg secret_access_key "${aws_secret_access_key}" \
--arg bucket_name "${bucket_name}" \
--arg region "${aws_region}" \
'{
access_key_id: $access_key_id,
secret_access_key: $secret_access_key,
bucket_name: $bucket_name,
region: $region
}' > "${CONFIG_FILE}"
;;

gcs)
: "${google_json_key_data:?}"

# login via gcloud
source "${script_dir}/../gcs/utils.sh"
gcloud_login

# create the regional bucket
make -C "${script_dir}/../gcs" regional-bucket

# read bucket name from the lock file
bucket_name="$(read_bucket_name "regional")"
: "${bucket_name:?failed to read bucket name from gcs lock file}"

jq -n \
--arg json_key "${google_json_key_data}" \
--arg bucket_name "${bucket_name}" \
'{
provider: "Google",
credentials_source: "static",
json_key: $json_key,
bucket_name: $bucket_name,
uniform_bucket_level_access: true
}' > "${CONFIG_FILE}"
;;

azurebs)
: "${azure_storage_account:?}"
: "${azure_storage_key:?}"
export environment="AzureCloud"

# create the container
bash "${script_dir}/../azurebs/setup.sh"

# read the container name from the lock file
source "${script_dir}/../azurebs/utils.sh"
container_name="$(read_container_name_from_file "${environment}")"
: "${container_name:?failed to read container name from lock file}"

jq -n \
--arg account_name "${azure_storage_account}" \
--arg account_key "${azure_storage_key}" \
--arg container_name "${container_name}" \
'{
account_name: $account_name,
account_key: $account_key,
container_name: $container_name
}' > "${CONFIG_FILE}"
;;

alioss)
: "${alioss_access_key_id:?}"
: "${alioss_access_key_secret:?}"
: "${alioss_region:=eu-central-1}"

# translate to names the alioss script expects
export access_key_id="${alioss_access_key_id}"
export access_key_secret="${alioss_access_key_secret}"
export region="${alioss_region}"
export test_name="benchmark"

# create the bucket
bash "${script_dir}/../alioss/setup.sh"

# read the bucket name from the lock file
source "${script_dir}/../alioss/utils.sh"
bucket_name="$(read_bucket_name_from_file "${test_name}")"
: "${bucket_name:?failed to read bucket name from alioss lock file}"

jq -n \
--arg access_key_id "${alioss_access_key_id}" \
--arg access_key_secret "${alioss_access_key_secret}" \
--arg bucket_name "${bucket_name}" \
--arg endpoint "oss-${alioss_region}.aliyuncs.com" \
'{
access_key_id: $access_key_id,
access_key_secret: $access_key_secret,
bucket_name: $bucket_name,
endpoint: $endpoint
}' > "${CONFIG_FILE}"
;;

*)
echo "Unknown provider: ${provider}. Must be one of: s3, gcs, azurebs, alioss"
exit 1
;;
esac

echo "Setup complete. Config written to ${CONFIG_FILE}"
chmod 600 "${CONFIG_FILE}"
75 changes: 75 additions & 0 deletions .github/scripts/benchmark/teardown.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
set -euo pipefail

script_dir="$( cd "$(dirname "${0}")" && pwd )"

: "${benchmark_config_file:?}"
: "${provider:?provider is required (s3|gcs|azurebs|alioss)}"
CONFIG_FILE="${benchmark_config_file}"

# always remove sensitive files even if teardown fails
trap 'rm -f "${CONFIG_FILE}" "/tmp/gcp-key.json"' EXIT

case "${provider}" in

s3)
: "${aws_access_key_id:?}"
: "${aws_secret_access_key:?}"
: "${aws_region:=us-east-1}"
: "${aws_stack_name:=s3cli-benchmark-${GITHUB_RUN_ID:-local}}"

# translate to names the s3 teardown script expects
export access_key_id="${aws_access_key_id}"
export secret_access_key="${aws_secret_access_key}"
export region_name="${aws_region}"
export stack_name="${aws_stack_name}"

bash "${script_dir}/../s3/teardown-infrastructure.sh"
Comment thread
stephanme marked this conversation as resolved.
Resolved
;;

gcs)
: "${google_json_key_data:?}"

source "${script_dir}/../gcs/utils.sh"
gcloud_login

# read bucket name and delete — cannot use make clean-gcs since it
# requires all three lock files (regional, multiregional, public)
# but we only created the regional bucket
bucket_name="$(read_bucket_name "regional")"
: "${bucket_name:?failed to read bucket name from gcs lock file}"

if ! error=$(gsutil rm -r "gs://${bucket_name}" 2>&1); then
echo "::warning::failed to delete gs://${bucket_name}: ${error}"
fi
;;

azurebs)
: "${azure_storage_account:?}"
: "${azure_storage_key:?}"
export environment="AzureCloud"

bash "${script_dir}/../azurebs/teardown.sh"
;;

alioss)
: "${alioss_access_key_id:?}"
: "${alioss_access_key_secret:?}"
: "${alioss_region:=eu-central-1}"

# translate to names the alioss teardown script expects
export access_key_id="${alioss_access_key_id}"
export access_key_secret="${alioss_access_key_secret}"
export region="${alioss_region}"
export test_name="benchmark"

bash "${script_dir}/../alioss/teardown.sh"
;;

*)
echo "Unknown provider: ${provider}. Must be one of: s3, gcs, azurebs, alioss"
exit 1
;;
esac

echo "Teardown complete."
8 changes: 8 additions & 0 deletions .github/scripts/gcs/utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -69,4 +69,12 @@ function gcloud_login {
echo "${google_json_key_data:-}" > "${keyfile}"
gcloud auth activate-service-account --key-file="${keyfile}"
gcloud config set project "${google_project:-}"
}


function read_bucket_name {
# mirrors TMP_DIR in .github/scripts/gcs/Makefile
local gcs_tmp_dir="/tmp/storage-cli-gcs-${GITHUB_RUN_ID:-${USER}}"
local storage_class="${1:?storage class required}"
cat "${gcs_tmp_dir}/${storage_class}.lock"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"Resources": {
"S3BenchmarkBucket": {
"Type": "AWS::S3::Bucket",
"DeletionPolicy": "Delete",
"Properties": {
"PublicAccessBlockConfiguration": {
"BlockPublicAcls": true,
"BlockPublicPolicy": true,
"IgnorePublicAcls": true,
"RestrictPublicBuckets": true
}
}
}
},
"Outputs": {
"BucketName": {
"Value": {
"Ref": "S3BenchmarkBucket"
}
}
}
}
3 changes: 2 additions & 1 deletion .github/scripts/s3/assets/lambda_function.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@ def test_runner_handler(event, context):
['./integration.test', '-ginkgo.label-filter', label_filter],
env=os.environ,
stderr=subprocess.STDOUT,
text=True,
)
logger.debug("INTEGRATION TEST OUTPUT:")
logger.debug(output)
except subprocess.CalledProcessError as e:
logger.debug("INTEGRATION TEST EXITED WITH STATUS: " + str(e.returncode))
logger.debug(e.output)
raise
raise
Loading
Loading