diff --git a/.github/scripts/benchmark/setup.sh b/.github/scripts/benchmark/setup.sh new file mode 100755 index 00000000..cd0bb263 --- /dev/null +++ b/.github/scripts/benchmark/setup.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$( cd "$(dirname "${0}")" && pwd )" + + +: "${benchmark_config_file:?}" +: "${provider:?provider is required (s3|gcs|azurebs|alioss)}" +CONFIG_FILE="${benchmark_config_file}" + +case "${provider}" in + + s3) + : "${aws_access_key_id:?}" + : "${aws_secret_access_key:?}" + : "${aws_region:=us-east-1}" + : "${aws_stack_name:=s3cli-benchmark-${GITHUB_RUN_ID:-local}}" + + # translate to names the s3 setup-aws-infrastructure.sh script expects + export access_key_id="${aws_access_key_id}" + export secret_access_key="${aws_secret_access_key}" + export region_name="${aws_region}" + export stack_name="${aws_stack_name}" + export template_name="s3cli-benchmark" + + bash "${script_dir}/../s3/setup-aws-infrastructure.sh" + + # source utils to use get_stack_info functions + # re-export AWS vars since subprocess env does not propagate back + export AWS_ACCESS_KEY_ID="${aws_access_key_id}" + export AWS_SECRET_ACCESS_KEY="${aws_secret_access_key}" + export AWS_DEFAULT_REGION="${aws_region}" + source "${script_dir}/../s3/utils.sh" + stack_info=$(get_stack_info "${aws_stack_name}") + bucket_name=$(get_stack_info_of "${stack_info}" "BucketName") + : "${bucket_name:?failed to read bucket name from stack outputs}" + + jq -n \ + --arg access_key_id "${aws_access_key_id}" \ + --arg secret_access_key "${aws_secret_access_key}" \ + --arg bucket_name "${bucket_name}" \ + --arg region "${aws_region}" \ + '{ + access_key_id: $access_key_id, + secret_access_key: $secret_access_key, + bucket_name: $bucket_name, + region: $region + }' > "${CONFIG_FILE}" + ;; + + gcs) + : "${google_json_key_data:?}" + + # login via gcloud + source "${script_dir}/../gcs/utils.sh" + gcloud_login + + # create the regional bucket + make -C "${script_dir}/../gcs" regional-bucket + + # read bucket name from the lock file + bucket_name="$(read_bucket_name "regional")" + : "${bucket_name:?failed to read bucket name from gcs lock file}" + + jq -n \ + --arg json_key "${google_json_key_data}" \ + --arg bucket_name "${bucket_name}" \ + '{ + provider: "Google", + credentials_source: "static", + json_key: $json_key, + bucket_name: $bucket_name, + uniform_bucket_level_access: true + }' > "${CONFIG_FILE}" + ;; + + azurebs) + : "${azure_storage_account:?}" + : "${azure_storage_key:?}" + export environment="AzureCloud" + + # create the container + bash "${script_dir}/../azurebs/setup.sh" + + # read the container name from the lock file + source "${script_dir}/../azurebs/utils.sh" + container_name="$(read_container_name_from_file "${environment}")" + : "${container_name:?failed to read container name from lock file}" + + jq -n \ + --arg account_name "${azure_storage_account}" \ + --arg account_key "${azure_storage_key}" \ + --arg container_name "${container_name}" \ + '{ + account_name: $account_name, + account_key: $account_key, + container_name: $container_name + }' > "${CONFIG_FILE}" + ;; + + alioss) + : "${alioss_access_key_id:?}" + : "${alioss_access_key_secret:?}" + : "${alioss_region:=eu-central-1}" + + # translate to names the alioss script expects + export access_key_id="${alioss_access_key_id}" + export access_key_secret="${alioss_access_key_secret}" + export region="${alioss_region}" + export test_name="benchmark" + + # create the bucket + bash "${script_dir}/../alioss/setup.sh" + + # read the bucket name from the lock file + source "${script_dir}/../alioss/utils.sh" + bucket_name="$(read_bucket_name_from_file "${test_name}")" + : "${bucket_name:?failed to read bucket name from alioss lock file}" + + jq -n \ + --arg access_key_id "${alioss_access_key_id}" \ + --arg access_key_secret "${alioss_access_key_secret}" \ + --arg bucket_name "${bucket_name}" \ + --arg endpoint "oss-${alioss_region}.aliyuncs.com" \ + '{ + access_key_id: $access_key_id, + access_key_secret: $access_key_secret, + bucket_name: $bucket_name, + endpoint: $endpoint + }' > "${CONFIG_FILE}" + ;; + + *) + echo "Unknown provider: ${provider}. Must be one of: s3, gcs, azurebs, alioss" + exit 1 + ;; +esac + +echo "Setup complete. Config written to ${CONFIG_FILE}" +chmod 600 "${CONFIG_FILE}" diff --git a/.github/scripts/benchmark/teardown.sh b/.github/scripts/benchmark/teardown.sh new file mode 100755 index 00000000..83213aba --- /dev/null +++ b/.github/scripts/benchmark/teardown.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$( cd "$(dirname "${0}")" && pwd )" + +: "${benchmark_config_file:?}" +: "${provider:?provider is required (s3|gcs|azurebs|alioss)}" +CONFIG_FILE="${benchmark_config_file}" + +# always remove sensitive files even if teardown fails +trap 'rm -f "${CONFIG_FILE}" "/tmp/gcp-key.json"' EXIT + +case "${provider}" in + + s3) + : "${aws_access_key_id:?}" + : "${aws_secret_access_key:?}" + : "${aws_region:=us-east-1}" + : "${aws_stack_name:=s3cli-benchmark-${GITHUB_RUN_ID:-local}}" + + # translate to names the s3 teardown script expects + export access_key_id="${aws_access_key_id}" + export secret_access_key="${aws_secret_access_key}" + export region_name="${aws_region}" + export stack_name="${aws_stack_name}" + + bash "${script_dir}/../s3/teardown-infrastructure.sh" + ;; + + gcs) + : "${google_json_key_data:?}" + + source "${script_dir}/../gcs/utils.sh" + gcloud_login + + # read bucket name and delete — cannot use make clean-gcs since it + # requires all three lock files (regional, multiregional, public) + # but we only created the regional bucket + bucket_name="$(read_bucket_name "regional")" + : "${bucket_name:?failed to read bucket name from gcs lock file}" + + if ! error=$(gsutil rm -r "gs://${bucket_name}" 2>&1); then + echo "::warning::failed to delete gs://${bucket_name}: ${error}" + fi + ;; + + azurebs) + : "${azure_storage_account:?}" + : "${azure_storage_key:?}" + export environment="AzureCloud" + + bash "${script_dir}/../azurebs/teardown.sh" + ;; + + alioss) + : "${alioss_access_key_id:?}" + : "${alioss_access_key_secret:?}" + : "${alioss_region:=eu-central-1}" + + # translate to names the alioss teardown script expects + export access_key_id="${alioss_access_key_id}" + export access_key_secret="${alioss_access_key_secret}" + export region="${alioss_region}" + export test_name="benchmark" + + bash "${script_dir}/../alioss/teardown.sh" + ;; + + *) + echo "Unknown provider: ${provider}. Must be one of: s3, gcs, azurebs, alioss" + exit 1 + ;; +esac + +echo "Teardown complete." diff --git a/.github/scripts/gcs/utils.sh b/.github/scripts/gcs/utils.sh index ba4f5a5c..3f505507 100644 --- a/.github/scripts/gcs/utils.sh +++ b/.github/scripts/gcs/utils.sh @@ -69,4 +69,12 @@ function gcloud_login { echo "${google_json_key_data:-}" > "${keyfile}" gcloud auth activate-service-account --key-file="${keyfile}" gcloud config set project "${google_project:-}" +} + + +function read_bucket_name { + # mirrors TMP_DIR in .github/scripts/gcs/Makefile + local gcs_tmp_dir="/tmp/storage-cli-gcs-${GITHUB_RUN_ID:-${USER}}" + local storage_class="${1:?storage class required}" + cat "${gcs_tmp_dir}/${storage_class}.lock" } \ No newline at end of file diff --git a/.github/scripts/s3/assets/cloudformation-s3cli-benchmark.template.json b/.github/scripts/s3/assets/cloudformation-s3cli-benchmark.template.json new file mode 100644 index 00000000..0b81da5f --- /dev/null +++ b/.github/scripts/s3/assets/cloudformation-s3cli-benchmark.template.json @@ -0,0 +1,23 @@ +{ + "Resources": { + "S3BenchmarkBucket": { + "Type": "AWS::S3::Bucket", + "DeletionPolicy": "Delete", + "Properties": { + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": true, + "IgnorePublicAcls": true, + "RestrictPublicBuckets": true + } + } + } + }, + "Outputs": { + "BucketName": { + "Value": { + "Ref": "S3BenchmarkBucket" + } + } + } +} diff --git a/.github/scripts/s3/assets/lambda_function.py b/.github/scripts/s3/assets/lambda_function.py index 2f0f2018..d9a350c4 100644 --- a/.github/scripts/s3/assets/lambda_function.py +++ b/.github/scripts/s3/assets/lambda_function.py @@ -19,10 +19,11 @@ def test_runner_handler(event, context): ['./integration.test', '-ginkgo.label-filter', label_filter], env=os.environ, stderr=subprocess.STDOUT, + text=True, ) logger.debug("INTEGRATION TEST OUTPUT:") logger.debug(output) except subprocess.CalledProcessError as e: logger.debug("INTEGRATION TEST EXITED WITH STATUS: " + str(e.returncode)) logger.debug(e.output) - raise + raise \ No newline at end of file diff --git a/.github/scripts/s3/run-integration-aws-iam.sh b/.github/scripts/s3/run-integration-aws-iam.sh index e7cc6775..52020c83 100755 --- a/.github/scripts/s3/run-integration-aws-iam.sh +++ b/.github/scripts/s3/run-integration-aws-iam.sh @@ -1,12 +1,33 @@ #!/usr/bin/env bash -set -euo pipefail +# IAM Role Integration Test Flow: +# +# 1. Retrieve the S3 bucket name and IAM role ARN from the CloudFormation stack. +# +# 2. Build storage-cli and Ginkgo integration tests, then package them +# with the Python Lambda handler into a ZIP file. +# +# 3. Create an AWS Lambda function using the ZIP and assign the IAM +# execution role from CloudFormation. +# +# 4. Invoke Lambda synchronously. The Python handler runs Ginkgo tests, +# which verify that storage-cli can access S3 using temporary IAM +# role credentials provided through the Lambda environment. +# +# 5. Check the Lambda invocation response (StatusCode and FunctionError) +# to determine whether the integration tests passed or failed. +# +# 6. Retrieve and print CloudWatch logs to show Ginkgo test output, +# including failed assertions and debugging information. +# +# 7. On script exit, the EXIT trap attempts to delete the Lambda function +# and its CloudWatch log group, warning if cleanup cannot be confirmed. + +set -euo pipefail -# Get the directory where this script is located -script_dir="$( cd "$(dirname "${0}")" && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd "${script_dir}/../../.." && pwd)" -# Source utils from the same directory source "${script_dir}/utils.sh" : "${access_key_id:?}" @@ -15,37 +36,64 @@ source "${script_dir}/utils.sh" : "${stack_name:?}" : "${label_filter:?}" -# Just need these to get the stack info and to create/invoke the Lambda function -export AWS_ACCESS_KEY_ID=${access_key_id} -export AWS_SECRET_ACCESS_KEY=${secret_access_key} -export AWS_DEFAULT_REGION=${region_name} +export AWS_ACCESS_KEY_ID="${access_key_id}" +export AWS_SECRET_ACCESS_KEY="${secret_access_key}" +export AWS_REGION="${region_name}" +export AWS_DEFAULT_REGION="${region_name}" +# Read CloudFormation outputs stack_info=$(get_stack_info "${stack_name}") bucket_name=$(get_stack_info_of "${stack_info}" "BucketName") iam_role_arn=$(get_stack_info_of "${stack_info}" "IamRoleArn") -# Create JSON payload and base64 encode it -lambda_payload_json="{\"region\": \"${region_name}\", \"bucket_name\": \"${bucket_name}\", \"s3_host\": \"s3.amazonaws.com\", \"label_filter\": \"${label_filter}\"}" -lambda_payload_base64=$(echo -n "${lambda_payload_json}" | base64) - +# Validate required stack outputs +[[ -n "${bucket_name}" && "${bucket_name}" != "null" ]] +[[ -n "${iam_role_arn}" && "${iam_role_arn}" != "null" ]] + +# Prepare Lambda payload +lambda_payload_json=$(jq -nc \ + --arg region "${region_name}" \ + --arg bucket "${bucket_name}" \ + --arg filter "${label_filter}" \ + '{ + region: $region, + bucket_name: $bucket, + s3_host: "s3.amazonaws.com", + label_filter: $filter + }') + +lambda_payload_base64=$( + printf '%s' "${lambda_payload_json}" | base64 | tr -d '\n' +) + +# Register cleanup before creating AWS resources +lambda_function_name="s3cli-integration-$(date +%s)-${RANDOM}" +log_group_name="/aws/lambda/${lambda_function_name}" lambda_log=$(mktemp -t "XXXXXX-lambda.log") -trap 'cat "${lambda_log}"' EXIT -# Go to the repository root (3 levels up from script directory) +trap cleanup_lambda EXIT + +cd "${repo_root}" + +# Build test artifacts +export CGO_ENABLED=0 +export GOOS=linux +export GOARCH=amd64 -pushd "${repo_root}" > /dev/null - export CGO_ENABLED=0 - export GOOS=linux - export GOARCH=amd64 - echo -e "\n building artifact with $(go version)..." - go build -o out/s3cli - ginkgo build s3/integration +echo "Building integration test artifacts..." - zip -j payload.zip s3/integration/integration.test out/s3cli ${script_dir}/assets/lambda_function.py +go build -o out/s3cli +ginkgo build s3/integration - lambda_function_name=s3cli-integration-$(date +%s) +zip -j payload.zip \ + s3/integration/integration.test \ + out/s3cli \ + "${script_dir}/assets/lambda_function.py" - aws lambda create-function \ +# Create Lambda +echo "Creating Lambda: ${lambda_function_name}" + +aws lambda create-function \ --region "${region_name}" \ --function-name "${lambda_function_name}" \ --zip-file fileb://payload.zip \ @@ -53,65 +101,104 @@ pushd "${repo_root}" > /dev/null --timeout 600 \ --memory-size 512 \ --handler lambda_function.test_runner_handler \ - --runtime python3.9 - - echo "Create done, invoking Lambda function ${lambda_function_name}..." - set +e - tries=0 - get_function_status_command="aws lambda get-function --region ${region_name} --function-name ${lambda_function_name}" - function_status=$(${get_function_status_command}) - while [[ ( $(echo "${function_status}" | jq -r ".Configuration.State") != "Active" ) && ( $tries -ne 15 ) ]] ; do - sleep 3 - echo "Checking for function readiness; attempt: $tries" - tries=$((tries + 1)) - function_status=$(${get_function_status_command}) - done - set -e - - aws lambda invoke \ - --invocation-type RequestResponse \ - --function-name "${lambda_function_name}" \ - --region "${region_name}" \ - --log-type Tail \ - --payload "${lambda_payload_base64}" \ - "${lambda_log}" | tee lambda_output.json + --runtime python3.9 > /dev/null - set +e - log_group_name="/aws/lambda/${lambda_function_name}" +# Wait until Lambda is ready +echo "Waiting for Lambda to become active..." - logs_command="aws logs describe-log-streams --log-group-name=${log_group_name}" - tries=0 - - log_streams_json=$(${logs_command}) - while [[ ( $? -ne 0 ) && ( $tries -ne 5 ) ]] ; do - sleep 2 - echo "Retrieving CloudWatch logs; attempt: $tries" - tries=$((tries + 1)) - log_streams_json=$(${logs_command}) - done - set -e - - log_stream_name=$(echo "${log_streams_json}" | jq -r ".logStreams[0].logStreamName") - - echo "Lambda execution log output for ${log_stream_name}" - - tries=0 - : > lambda_output.log - while [[ ( "$(du lambda_output.log | cut -f 1)" -eq "0" ) && ( $tries -ne 20 ) ]] ; do - sleep 2 - tries=$((tries + 1)) - echo "Retrieving CloudWatch events; attempt: $tries" - - aws logs get-log-events \ - --log-group-name="${log_group_name}" \ - --log-stream-name="${log_stream_name}" \ - | jq -r ".events | map(.message) | .[]" | tee lambda_output.log - done - - aws lambda delete-function \ +aws lambda wait function-active-v2 \ + --region "${region_name}" \ --function-name "${lambda_function_name}" - aws logs delete-log-group --log-group-name="${log_group_name}" +# Invoke synchronously +echo "Invoking Lambda integration tests..." - jq -r ".FunctionError" < lambda_output.json | grep -v -e "Handled" -e "Unhandled" -popd > /dev/null +aws lambda invoke \ + --region "${region_name}" \ + --function-name "${lambda_function_name}" \ + --invocation-type RequestResponse \ + --log-type Tail \ + --payload "${lambda_payload_base64}" \ + --cli-read-timeout 900 \ + "${lambda_log}" > lambda_output.json + +echo "Lambda invocation response:" +cat "${lambda_log}" +echo +echo "Lambda invocation completed." + +# Check the invocation response +if jq -e '.StatusCode == 200 and .FunctionError == null' lambda_output.json > /dev/null; then + echo "Lambda execution succeeded." + test_failed=false +else + echo "::error::Lambda reported an execution error." + cat "${lambda_log}" + test_failed=true +fi + +# Retrieve CloudWatch logs +echo "Retrieving CloudWatch logs..." + +# Create an empty log file, or truncate it if it already exists +: > lambda_output.log + +max_attempts=30 +stable_count=0 +logs_complete=false + +for ((attempt=1; attempt<=max_attempts; attempt++)); do + echo "CloudWatch retrieval attempt ${attempt}/${max_attempts}" + + # Fetch all currently available pages across the log group. + # The Lambda function name makes this log group unique. + if aws logs filter-log-events \ + --region "${region_name}" \ + --log-group-name "${log_group_name}" \ + --output json > cloudwatch_response.json 2> cloudwatch_error.log; then + + jq -r '.events[].message' \ + cloudwatch_response.json > lambda_output.new + + # Require a REPORT marker and two consecutive + # identical snapshots after it appears. + if grep -q 'REPORT RequestId:' lambda_output.new; then + if cmp -s lambda_output.log lambda_output.new; then + stable_count=$((stable_count + 1)) + else + stable_count=0 + fi + else + stable_count=0 + fi + + mv lambda_output.new lambda_output.log + + if (( stable_count >= 2 )); then + logs_complete=true + break + fi + else + echo "CloudWatch logs not available yet:" + cat cloudwatch_error.log + fi + + sleep 2 +done + +echo "========== Lambda execution logs ==========" +cat lambda_output.log +echo "===========================================" + +if [[ "${logs_complete}" != true ]]; then + echo "::warning title=Incomplete Lambda Logs::Could not retrieve all Lambda execution logs. \ + Some debugging information may be missing. \ + Function: '${lambda_function_name}', Region: '${region_name}'." +fi + +if [[ "${test_failed}" == true ]]; then + echo "::error title=Integration Test Failed::Lambda '${lambda_function_name}' reported an execution error." + exit 1 +fi + +echo "IAM integration tests completed successfully." diff --git a/.github/scripts/s3/setup-aws-infrastructure.sh b/.github/scripts/s3/setup-aws-infrastructure.sh index e60b40ec..195eb070 100755 --- a/.github/scripts/s3/setup-aws-infrastructure.sh +++ b/.github/scripts/s3/setup-aws-infrastructure.sh @@ -11,6 +11,7 @@ source "${script_dir}/utils.sh" : "${secret_access_key:?}" : "${region_name:?}" : "${stack_name:?}" +: "${template_name:=${stack_name}}" # defaults to stack_name for backward compatibility with existing callers (e.g. integration tests) export AWS_ACCESS_KEY_ID=${access_key_id} export AWS_SECRET_ACCESS_KEY=${secret_access_key} @@ -31,7 +32,7 @@ fi cmd="aws cloudformation create-stack \ --stack-name ${stack_name} \ - --template-body file://${script_dir}/assets/cloudformation-${stack_name}.template.json \ + --template-body file://${script_dir}/assets/cloudformation-${template_name}.template.json \ --capabilities CAPABILITY_IAM" echo "Running: ${cmd}"; ${cmd} diff --git a/.github/scripts/s3/teardown-infrastructure.sh b/.github/scripts/s3/teardown-infrastructure.sh index 0054cbba..ffd2dff1 100755 --- a/.github/scripts/s3/teardown-infrastructure.sh +++ b/.github/scripts/s3/teardown-infrastructure.sh @@ -37,8 +37,3 @@ while true; do exit 1 fi done - -echo "Deleting lambda functions" -aws lambda list-functions \ - | jq -r '.Functions[].FunctionName' \ - | xargs -n1 -I{} aws lambda delete-function --function-name {} diff --git a/.github/scripts/s3/utils.sh b/.github/scripts/s3/utils.sh index a58b0738..02315ca8 100755 --- a/.github/scripts/s3/utils.sh +++ b/.github/scripts/s3/utils.sh @@ -18,3 +18,70 @@ get_stack_status() { get_stack_info "${stack_name}" | jq -r '.StackStatus' } + + +cleanup_lambda() { +local exit_code=$? +echo "Cleaning up Lambda: ${lambda_function_name}" + +local deleted=false +local delete_accepted=false +local error + +if error=$(aws lambda delete-function \ + --region "${region_name}" \ + --function-name "${lambda_function_name}" 2>&1); then + delete_accepted=true +elif [[ "${error}" == *"ResourceNotFoundException"* ]]; then + deleted=true +else + echo "Initial Lambda deletion failed: ${error}" +fi + +for attempt in {1..5}; do + [[ "${deleted}" == true ]] && break + + if error=$(aws lambda get-function \ + --region "${region_name}" \ + --function-name "${lambda_function_name}" 2>&1); then + + echo "Lambda still exists (check ${attempt}/5)." + + # Retry deletion only if no previous request succeeded + if [[ "${delete_accepted}" == false ]]; then + if aws lambda delete-function \ + --region "${region_name}" \ + --function-name "${lambda_function_name}"; then + delete_accepted=true + fi + fi + + elif [[ "${error}" == *"ResourceNotFoundException"* ]]; then + echo "Lambda deletion confirmed." + deleted=true + break + + else + echo "Unable to verify Lambda state: ${error}" + fi + + if [[ "${attempt}" -lt 5 ]]; then + sleep 3 + fi +done + + +if [[ "${deleted}" == false ]]; then + echo "::warning title=Potential AWS Resource Leak::Lambda '${lambda_function_name}' in '${region_name}' may still exist. \ +Automatic cleanup failed or could not be verified." +fi + + +aws logs delete-log-group \ + --region "${region_name}" \ + --log-group-name "/aws/lambda/${lambda_function_name}" || true + +rm -f "${lambda_log}" + +exit "${exit_code}" +} diff --git a/.github/workflows/benchmark-manual.yml b/.github/workflows/benchmark-manual.yml new file mode 100644 index 00000000..1739aa9f --- /dev/null +++ b/.github/workflows/benchmark-manual.yml @@ -0,0 +1,108 @@ +name: Benchmark Manual + +on: + workflow_dispatch: + inputs: + profile: + description: "Benchmark profile" + required: true + type: choice + default: quick + options: + - quick + - full + provider: + description: "Storage provider" + required: true + type: choice + default: s3 + options: + - s3 + - gcs + - azurebs + - alioss + +jobs: + benchmarking: + name: Run Benchmarking + runs-on: [ubuntu-latest] + env: + BENCHMARK_CONFIG_FILE: /tmp/benchmark-config.json + steps: + - name: Checkout Into Source Code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + + - name: Build benchmarker + run: | + echo "Building Benchmarker." + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -tags benchmark -o "benchmarker" ./benchmark/ + + - name: Build storage-cli + run: | + echo "Building Storage CLI." + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X main.version=0.0.0" -o "storage-cli" . + + - name: Set up Aliyun CLI + if: inputs.provider == 'alioss' + uses: aliyun/setup-aliyun-cli-action@v1 + + - name: Setup The Environment + run: | + export aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + echo "${{ secrets.GCP_SERVICE_ACCOUNT_BASE64 }}" | base64 -d > /tmp/gcp-key.json + export google_json_key_data="$(cat /tmp/gcp-key.json)" + export azure_storage_account="${{ secrets.AZURE_STORAGE_ACCOUNT }}" + export azure_storage_key="${{ secrets.AZURE_STORAGE_KEY }}" + export alioss_access_key_id="${{ secrets.ALI_ACCESS_KEY_ID }}" + export alioss_access_key_secret="${{ secrets.ALI_ACCESS_KEY_SECRET }}" + export provider="${{ inputs.provider }}" + export benchmark_config_file="${{ env.BENCHMARK_CONFIG_FILE }}" + ./.github/scripts/benchmark/setup.sh + + - name: Run Benchmark + run: | + mkdir -p "${GITHUB_WORKSPACE}/benchmark-results" + ./benchmarker \ + -candidate ./storage-cli \ + -provider "${{ inputs.provider }}" \ + -config "${{ env.BENCHMARK_CONFIG_FILE }}" \ + -profile "${{ inputs.profile }}" \ + -output "${GITHUB_WORKSPACE}/benchmark-results" + + - name: Upload Artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: benchmark-results-${{ inputs.provider }}-${{ inputs.profile }} + path: ${{ github.workspace }}/benchmark-results/ + retention-days: 90 + + - name: Append Summary + if: always() + run: | + if [ -f "${GITHUB_WORKSPACE}/benchmark-results/summary.md" ]; then + cat "${GITHUB_WORKSPACE}/benchmark-results/summary.md" >> "${GITHUB_STEP_SUMMARY}" + else + echo "No summary.md produced (benchmark did not complete)." >> "${GITHUB_STEP_SUMMARY}" + fi + + - name: Teardown The Environment + if: always() + run: | + export aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + echo "${{ secrets.GCP_SERVICE_ACCOUNT_BASE64 }}" | base64 -d > /tmp/gcp-key.json + export google_json_key_data="$(cat /tmp/gcp-key.json)" + export azure_storage_account="${{ secrets.AZURE_STORAGE_ACCOUNT }}" + export azure_storage_key="${{ secrets.AZURE_STORAGE_KEY }}" + export alioss_access_key_id="${{ secrets.ALI_ACCESS_KEY_ID }}" + export alioss_access_key_secret="${{ secrets.ALI_ACCESS_KEY_SECRET }}" + export provider="${{ inputs.provider }}" + export benchmark_config_file="${{ env.BENCHMARK_CONFIG_FILE }}" + ./.github/scripts/benchmark/teardown.sh \ No newline at end of file diff --git a/.github/workflows/release-cron.yml b/.github/workflows/release-cron.yml index 840a7481..b4531892 100644 --- a/.github/workflows/release-cron.yml +++ b/.github/workflows/release-cron.yml @@ -116,6 +116,8 @@ jobs: needs: [check, tests, integration-s3, integration-gcs, integration-azurebs, integration-alioss, integration-dav] if: needs.tests.result == 'success' runs-on: ubuntu-latest + outputs: + version_number: ${{ steps.version.outputs.VERSION_NUMBER }} steps: - name: Checkout Into Source Code uses: actions/checkout@v7 @@ -201,3 +203,91 @@ jobs: draft: false makeLatest: true generateReleaseNotes: true + + benchmark: + name: Performance Benchmark - ${{ matrix.provider }} + needs: [build-and-release] + if: needs.build-and-release.result == 'success' + runs-on: ubuntu-latest + env: + BENCHMARK_CONFIG_FILE: /tmp/benchmark-config.json + strategy: + fail-fast: false + matrix: + provider: [s3, gcs, azurebs, alioss] + steps: + - name: Checkout Into Source Code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + + - name: Set up Aliyun CLI + if: matrix.provider == 'alioss' + uses: aliyun/setup-aliyun-cli-action@v1 + + - name: Build benchmarker + run: | + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -tags benchmark -o "benchmarker" ./benchmark/ + + - name: Build storage-cli + run: | + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X main.version=${{ needs.build-and-release.outputs.version_number }}" -o "storage-cli" . + + - name: Setup The Environment + run: | + export aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + echo "${{ secrets.GCP_SERVICE_ACCOUNT_BASE64 }}" | base64 -d > /tmp/gcp-key.json + export google_json_key_data="$(cat /tmp/gcp-key.json)" + export azure_storage_account="${{ secrets.AZURE_STORAGE_ACCOUNT }}" + export azure_storage_key="${{ secrets.AZURE_STORAGE_KEY }}" + export alioss_access_key_id="${{ secrets.ALI_ACCESS_KEY_ID }}" + export alioss_access_key_secret="${{ secrets.ALI_ACCESS_KEY_SECRET }}" + export provider="${{ matrix.provider }}" + export benchmark_config_file="${{ env.BENCHMARK_CONFIG_FILE }}" + ./.github/scripts/benchmark/setup.sh + + - name: Run Benchmark + run: | + mkdir -p "${GITHUB_WORKSPACE}/benchmark-results" + ./benchmarker \ + -candidate ./storage-cli \ + -provider "${{ matrix.provider }}" \ + -config "${BENCHMARK_CONFIG_FILE}" \ + -profile full \ + -output "${GITHUB_WORKSPACE}/benchmark-results" + + - name: Upload Artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: benchmark-results-${{ matrix.provider }}-full + path: ${{ github.workspace }}/benchmark-results/ + retention-days: 90 + + - name: Append Summary + if: always() + run: | + if [ -f "${GITHUB_WORKSPACE}/benchmark-results/summary.md" ]; then + cat "${GITHUB_WORKSPACE}/benchmark-results/summary.md" >> "${GITHUB_STEP_SUMMARY}" + else + echo "No summary.md produced (benchmark did not complete)." >> "${GITHUB_STEP_SUMMARY}" + fi + + - name: Teardown The Environment + if: always() + run: | + export aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" + export aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export azure_storage_account="${{ secrets.AZURE_STORAGE_ACCOUNT }}" + export azure_storage_key="${{ secrets.AZURE_STORAGE_KEY }}" + export alioss_access_key_id="${{ secrets.ALI_ACCESS_KEY_ID }}" + export alioss_access_key_secret="${{ secrets.ALI_ACCESS_KEY_SECRET }}" + echo "${{ secrets.GCP_SERVICE_ACCOUNT_BASE64 }}" | base64 -d > /tmp/gcp-key.json + export google_json_key_data="$(cat /tmp/gcp-key.json)" + export provider="${{ matrix.provider }}" + export benchmark_config_file="${{ env.BENCHMARK_CONFIG_FILE }}" + ./.github/scripts/benchmark/teardown.sh diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 89c141c7..cbcaabf9 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -61,4 +61,10 @@ jobs: export CGO_ENABLED=0 go version go run github.com/onsi/ginkgo/v2/ginkgo ./common/... + + - name: benchmark unit tests + run: | + export CGO_ENABLED=0 + go version + go run github.com/onsi/ginkgo/v2/ginkgo --tags benchmark ./benchmark/... \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index fa734787..58669b9c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,9 @@ version: "2" +run: + build-tags: + - benchmark + linters: default: standard diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..4403463a --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,95 @@ +# Benchmarker +Standalone benchmark harness for storage-cli. Measures wall-clock transfer performance of the CLI binary against real provider buckets. + +## How it works +The harness invokes the real `storage-cli` binary as a subprocess for every `put` and `get`, measures elapsed wall-clock time, and verifies downloaded content against a SHA-256 computed before the timer starts. This exercises flags, config parsing, client construction, SDK behaviour, and exit status at exactly the CLI boundary used by Cloud Controller. + +Failed or timed-out samples are retained in the output and mark the run as failed; they never appear as successful timing samples. + +## Test Profiles +| Profile | Object sizes | Samples | When to use | +|---------|-------------|---------|-------------| +| `quick` | 1 KiB, 1 MiB, 10 MiB, 128 MiB | 3 measured round trips per size | Fast sanity check on relevant changes — covers latency-dominated (1 KiB) through large-object (128 MiB) transfers without a warm-up | +| `full` | 1 KiB, 1 MiB, 10 MiB, 128 MiB, 1 GiB | 5 measured round trips per size | Weekly runs, pre-release validation, and investigations — the extra 1 GiB size exercises transfers well beyond provider multipart/chunk thresholds (Azure 32 MiB, GCS 100 MiB, S3 5 MiB parts) | + +## Usage +```sh +# Build the harness +go build -tags benchmark -o benchmarker ./benchmark + +# Quick run against AWS S3 +./benchmarker \ + -candidate ./storage-cli-linux-amd64 \ + -provider s3 \ + -config /path/to/config-s3.json \ + -profile quick \ + -output ./results/s3 + +# Full run with baseline comparison against AWS ceph +./benchmarker \ + -candidate ./storage-cli-linux-amd64 \ + -baseline ./storage-cli-1.0.12-linux-amd64 \ + -provider s3 \ + -config /path/to/config-s3.json \ + -backend-label ceph \ + -profile full \ + -timeout 20m \ + -namespace benchmark/my-run \ + -output ./results/ceph +``` + +## Flags +| Flag | Default | Description | +|------|---------|-------------| +| `-candidate` | *(required)* | Path to the CLI binary under test | +| `-baseline` | *(none)* | Path to a previous release binary for comparison | +| `-provider` | *(required)* | CLI storage selector: `s3`, `gcs`, `azurebs`, `alioss` | +| `-config` | *(required)* | Path to provider JSON configuration file | +| `-backend-label` | *(same as -provider)* | Label used in reports to identify the actual storage backend. Defaults to the provider name. Set explicitly when the backend differs, e.g. `-provider s3 -backend-label ceph` | +| `-profile` | `quick` | `quick` or `full` | +| `-timeout` | `10m` | Per-operation timeout applied to each put and get. Must include a unit, e.g. `30s`, `10m`, `1h` | +| `-namespace` | `benchmark` | Top-level prefix for every object key written to the bucket, e.g. `benchmark/1731.../s3/candidate/1KiB/sample-0`. Used to group benchmark objects so bucket lifecycle rules and cleanup stay scoped away from other data in the same bucket. | +| `-run-id` | *(timestamp)* | Unique identifier embedded in all object keys | +| `-output` | `.` | Directory for output files | + +## Output Files +| File | Contents | +|------|----------| +| `results.json` | Raw per-sample timings, statuses, and metadata | +| `summary.md` | Markdown table with median latency and throughput; suitable for GitHub job summary | + +## Configuration Files +Provider configuration is read from the JSON file specified by `-config`. For the different providers, check the provider's specific README.md, e.g. s3/README.md + +Example structure for S3: + +```json +{ + "access_key_id": "...", + "secret_access_key": "...", + "bucket_name": "my-benchmark-bucket", + "region": "eu-central-1" +} +``` + +## Infrastructure Requirements +- Use a dedicated benchmark bucket or container, not a production or shared integration bucket. The harness deletes objects it creates, but lifecycle expiration should be configured as a backstop for interrupted runs. +- The IAM/service-account used for benchmarking needs only object-level read/write/delete on the dedicated bucket; no bucket-management permissions are required. Except the GCS whic requires storage.buckets.get permission. +- Object keys are namespaced as `///