Skip to content

[gce_test] Create InstallGcloudIfNeeded to migrate from gsutil to gcloud storage. #458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
franciscovalentecastro wants to merge 4 commits into
base: master
Choose a base branch
from
Open

[gce_test] Create InstallGcloudIfNeeded to migrate from gsutil to gcloud storage. #458

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8b09127
Create `InstallGcloudIfNeeded` to migrate from `gsutil` to `gcloud st…
franciscovalentecastro Dec 1, 2025
60086a0
Remove missing `gsutil` references.
franciscovalentecastro Dec 1, 2025
ba4db50
Keep custom gcloud setup.
franciscovalentecastro Dec 2, 2025
bcf2bbd
Link gcloud binary.
franciscovalentecastro Dec 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 18 additions & 18 deletions integration_test/gce-testing-internal/gce/gce_testing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -943,11 +943,11 @@ func UploadContent(ctx context.Context, logger *log.Logger, vm *VM, content io.R
_, err = RunRemotely(ctx, logger, vm, fmt.Sprintf(`New-Item -Path "%s" -ItemType File -Force ;Read-GcsObject -Force -Bucket "%s" -ObjectName "%s" -OutFile "%s"`, remotePath, object.BucketName(), object.ObjectName(), remotePath))
return err
}
if err := InstallGsutilIfNeeded(ctx, logger, vm); err != nil {
if err := InstallGcloudIfNeeded(ctx, logger, vm); err != nil {
return err
}
objectPath := fmt.Sprintf("gs://%s/%s", object.BucketName(), object.ObjectName())
_, err = RunRemotely(ctx, logger, vm, fmt.Sprintf("sudo gsutil cp '%s' '%s'", objectPath, remotePath))
_, err = RunRemotely(ctx, logger, vm, fmt.Sprintf("sudo gcloud storage cp '%s' '%s'", objectPath, remotePath))
return err
}

Expand Down Expand Up @@ -1939,7 +1939,7 @@ func InstallGrpcurlIfNeeded(ctx context.Context, logger *log.Logger, vm *VM) err
}

logger.Printf("grpcurl not found, installing it...")
installCmd := `gsutil cp gs://ops-agents-public-buckets-vendored-deps/mirrored-content/grpcurl/v1.8.6/grpcurl_1.8.6_windows_x86_64.zip C:\agentPlugin;Expand-Archive -Path "C:\agentPlugin\grpcurl_1.8.6_windows_x86_64.zip" -DestinationPath "C:\" -Force;ls "C:\"`
installCmd := `gcloud storage cp gs://ops-agents-public-buckets-vendored-deps/mirrored-content/grpcurl/v1.8.6/grpcurl_1.8.6_windows_x86_64.zip C:\agentPlugin;Expand-Archive -Path "C:\agentPlugin\grpcurl_1.8.6_windows_x86_64.zip" -DestinationPath "C:\" -Force;ls "C:\"`

_, err := RunRemotely(ctx, logger, vm, installCmd)
return err
Expand All @@ -1956,29 +1956,29 @@ func InstallGrpcurlIfNeeded(ctx context.Context, logger *log.Logger, vm *VM) err
arch = "arm64"
}

installCmd := fmt.Sprintf("sudo gsutil cp gs://ops-agents-public-buckets-vendored-deps/mirrored-content/grpcurl/v1.8.6/grpcurl_1.8.6_linux_%s.tar.gz /tmp/agentPlugin && sudo tar -xzf /tmp/agentPlugin/grpcurl_1.8.6_linux_%s.tar.gz --no-overwrite-dir -C /usr/local/bin", arch, arch)
installCmd := fmt.Sprintf("sudo gcloud storage cp gs://ops-agents-public-buckets-vendored-deps/mirrored-content/grpcurl/v1.8.6/grpcurl_1.8.6_linux_%s.tar.gz /tmp/agentPlugin && sudo tar -xzf /tmp/agentPlugin/grpcurl_1.8.6_linux_%s.tar.gz --no-overwrite-dir -C /usr/local/bin", arch, arch)
installCmd = `set -ex
` + installCmd
_, err := RunRemotely(ctx, logger, vm, installCmd)
return err
}

// InstallGsutilIfNeeded installs gsutil on instances that don't already have
// InstallGcloudIfNeeded installs gcloud cli on instances that don't already have
// it installed. This is only currently the case for some old versions of SUSE.
func InstallGsutilIfNeeded(ctx context.Context, logger *log.Logger, vm *VM) error {
func InstallGcloudIfNeeded(ctx context.Context, logger *log.Logger, vm *VM) error {
if IsWindows(vm.ImageSpec) {
return nil
}
if _, err := RunRemotely(ctx, logger, vm, "sudo gsutil --version"); err == nil {
// Success, no need to install gsutil.
if _, err := RunRemotely(ctx, logger, vm, "sudo gcloud --version"); err == nil {
// Success, no need to install gcloud.
return nil
}
logger.Printf("gsutil not found, installing it...")
logger.Printf("gcloud not found, installing it...")

// SUSE seems to be the only distro without gsutil, so what follows is all
// SUSE seems to be the only distro without gcloud, so what follows is all
// very SUSE-specific.
if !IsSUSEVM(vm) {
return installErr("gsutil", vm.OS.ID)
return installErr("gcloud", vm.OS.ID)
}

gcloudArch := "x86_64"
Expand Down Expand Up @@ -2007,15 +2007,15 @@ INSTALL_DIR="$(readlink --canonicalize .)"
# Upgrade to the latest version
sudo ${INSTALL_DIR}/google-cloud-sdk/bin/gcloud components update --quiet

sudo ln -s ${INSTALL_DIR}/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
sudo ln -s ${INSTALL_DIR}/google-cloud-sdk/bin/gcloud /usr/bin/gcloud
`
// b/308962066: The GCloud CLI ARM Linux tarballs do not have bundled Python
// and the GCloud CLI requires Python >= 3.8. Install Python311 for ARM VMs
if IsARM(vm.ImageSpec) {
// This is what's used on openSUSE.
repoSetupCmd := "sudo zypper --non-interactive refresh"
if strings.Contains(vm.ImageSpec, "sles-12") {
return installErr("gsutil", vm.ImageSpec)
return installErr("gcloud", vm.ImageSpec)
}
// For SLES 15 ARM: use a vendored repo to reduce flakiness of the
// external repos. See http://go/sdi/releases/build-test-release/vendored
Expand All @@ -2030,21 +2030,21 @@ sudo zypper --non-interactive refresh test-vendor`
` + repoSetupCmd + `
sudo zypper --non-interactive install python311 python3-certifi

# On SLES 15 and OpenSUSE Leap arm, python3 is Python 3.6. Tell gsutil/gcloud to use python3.11.
# On SLES 15 and OpenSUSE Leap arm, python3 is Python 3.6. Tell gcloud to use python3.11.
export CLOUDSDK_PYTHON=/usr/bin/python3.11

` + installFromTarball + `

# Upgrade to the latest version
sudo CLOUDSDK_PYTHON=/usr/bin/python3.11 ${INSTALL_DIR}/google-cloud-sdk/bin/gcloud components update --quiet

# Make a "gsutil" bash script in /usr/bin that runs the copy of gsutil that
# Make a "gcloud" bash script in /usr/bin that runs the copy of gcloud that
# was installed into $INSTALL_DIR with CLOUDSDK_PYTHON set.
sudo tee /usr/bin/gsutil > /dev/null << EOF
sudo tee /usr/bin/gcloud > /dev/null << EOF
#!/usr/bin/env bash
CLOUDSDK_PYTHON=/usr/bin/python3.11 ${INSTALL_DIR}/google-cloud-sdk/bin/gsutil "\$@"
CLOUDSDK_PYTHON=/usr/bin/python3.11 ${INSTALL_DIR}/google-cloud-sdk/bin/gcloud "\$@"
EOF
sudo chmod a+x /usr/bin/gsutil
sudo chmod a+x /usr/bin/gcloud
`
}

Expand Down
10 changes: 5 additions & 5 deletions integration_test/smoke_test/smoke_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ ZONES: comma-separated list of what zones to run in.
PROJECT: GCP project to run in.
IMAGE_SPECS: comma-separated list of image specs, see gce_testing.go for details.
OTELCOL_CONFIGS_DIR: path to config files for otelcol to use for testing.
_BUILD_ARTIFACTS_PACKAGE_GCS: gsutil URI for a directory containing otelcol-google
_BUILD_ARTIFACTS_PACKAGE_GCS: "gcloud storage" URI for a directory containing otelcol-google
package files. Should start with 'gs://'.
*/

Expand Down Expand Up @@ -178,7 +178,7 @@ func installWindowsPackageFromGCS(ctx context.Context, logger *log.Logger, vm *g
if _, err := gce.RunRemotely(ctx, logger, vm, "New-Item -ItemType directory -Path C:\\collectorUpload"); err != nil {
return err
}
if _, err := gce.RunRemotely(ctx, logger, vm, fmt.Sprintf("gsutil cp -r %s/*.goo C:\\collectorUpload", gcsPath)); err != nil {
if _, err := gce.RunRemotely(ctx, logger, vm, fmt.Sprintf("gcloud storage cp -r %s/*.goo C:\\collectorUpload", gcsPath)); err != nil {
return fmt.Errorf("error copying down collector package from GCS: %v", err)
}
if _, err := gce.RunRemotely(ctx, logger, vm, "googet -noconfirm -verbose install -reinstall (Get-ChildItem C:\\collectorUpload\\*.goo | Select-Object -Expand FullName)"); err != nil {
Expand All @@ -197,7 +197,7 @@ func installPackageFromGCS(ctx context.Context, logger *log.Logger, vm *gce.VM,
if _, err := gce.RunRemotely(ctx, logger, vm, "mkdir -p /tmp/collectorUpload"); err != nil {
return err
}
if err := gce.InstallGsutilIfNeeded(ctx, logger, vm); err != nil {
if err := gce.InstallGcloudIfNeeded(ctx, logger, vm); err != nil {
return err
}

Expand All @@ -220,7 +220,7 @@ func installPackageFromGCS(ctx context.Context, logger *log.Logger, vm *gce.VM,
ext = ".rpm"
}
pkgSelector := gcsPath + "/*" + arch + ext
if _, err := gce.RunRemotely(ctx, logger, vm, "sudo gsutil cp -r "+pkgSelector+" /tmp/collectorUpload"); err != nil {
if _, err := gce.RunRemotely(ctx, logger, vm, "sudo gcloud storage cp -r "+pkgSelector+" /tmp/collectorUpload"); err != nil {
return fmt.Errorf("error copying down collector package from GCS: %v", err)
}
// Print the contents of /tmp/collectorUpload into the logs.
Expand Down Expand Up @@ -293,7 +293,7 @@ func setupOtelCollectorFrom(ctx context.Context, logger *log.Logger, vm *gce.VM,
time.Sleep(10 * time.Second)

return nil
} // End windows handling.
} // End windows handling.

defaultConfigPath := collectorConfigPath(vm.ImageSpec)
if err := gce.UploadContent(ctx, logger, vm, strings.NewReader(config), defaultConfigPath); err != nil {
Expand Down