Merge pull request #3011 from googlyrahman/refactor-ai-migration-71846e7a93bb405fb1627f7831d4ba8c

WorkingGitHubUser · web-flow · commit b2153135e135 · 2025-11-10T12:31:19.000-07:00
Migrate gsutil usage to gcloud storage
diff --git a/quests/dataflow/create_batch_sinks.sh b/quests/dataflow/create_batch_sinks.sh
@@ -5,8 +5,8 @@ PROJECT_ID=$(gcloud config get-value project)
 
 # GCS buckets
 #TODO: Add try/catch for the first bucket since qwiklabs
-gsutil mb -l US gs://$PROJECT_ID
-gsutil mb -l US -c "COLDLINE" gs://$PROJECT_ID-coldline
+gcloud storage buckets create --location=US gs://$PROJECT_ID
+gcloud storage buckets create --location=US --default-storage-class="COLDLINE" gs://$PROJECT_ID-coldline
 
 # BiqQuery Dataset
 bq mk --location=US logs
diff --git a/quests/dataflow/create_streaming_sinks.sh b/quests/dataflow/create_streaming_sinks.sh
@@ -5,8 +5,8 @@ PROJECT_ID=$(gcloud config get-value project)
 
 # GCS buckets
 #TODO: Add try/catch for the first bucket since qwiklabs
-gsutil mb -l US gs://$PROJECT_ID
-gsutil mb -l US -c "COLDLINE" gs://$PROJECT_ID-coldline
+gcloud storage buckets create --location=US gs://$PROJECT_ID
+gcloud storage buckets create --location=US --default-storage-class="COLDLINE" gs://$PROJECT_ID-coldline
 
 # BiqQuery Dataset
 bq mk --location=US logs
diff --git a/quests/dataflow/generate_batch_events.sh b/quests/dataflow/generate_batch_events.sh
@@ -16,4 +16,4 @@ echo "Copying events to Cloud Storage"
 # Set BUCKET to the non-coldline Google Cloud Storage bucket
 export BUCKET=gs://$(gcloud config get-value project)/
 # Copy events.json into the bucket
-gsutil cp events.json ${BUCKET}
+gcloud storage cp events.json ${BUCKET}
diff --git a/quests/dataflow_python/create_batch_sinks.sh b/quests/dataflow_python/create_batch_sinks.sh
@@ -5,8 +5,8 @@ PROJECT_ID=$(gcloud config get-value project)
 
 # GCS buckets
 #TODO: Add try/catch for the first bucket since qwiklabs
-gsutil mb -l US gs://$PROJECT_ID
-gsutil mb -l US -c "COLDLINE" gs://$PROJECT_ID-coldline
+gcloud storage buckets create --location=US gs://$PROJECT_ID
+gcloud storage buckets create --location=US --default-storage-class="COLDLINE" gs://$PROJECT_ID-coldline
 
 # BiqQuery Dataset
 bq mk --location=US logs
diff --git a/quests/dataflow_python/create_streaming_sinks.sh b/quests/dataflow_python/create_streaming_sinks.sh
@@ -5,8 +5,8 @@ PROJECT_ID=$(gcloud config get-value project)
 
 # GCS buckets
 #TODO: Add try/catch for the first bucket since qwiklabs
-gsutil mb -l US gs://$PROJECT_ID
-gsutil mb -l US -c "COLDLINE" gs://$PROJECT_ID-coldline
+gcloud storage buckets create --location=US gs://$PROJECT_ID
+gcloud storage buckets create --location=US --default-storage-class="COLDLINE" gs://$PROJECT_ID-coldline
 
 # BiqQuery Dataset
 bq mk --location=US logs
diff --git a/quests/dataflow_python/generate_batch_events.sh b/quests/dataflow_python/generate_batch_events.sh
@@ -16,4 +16,4 @@ echo "Copying events to Cloud Storage"
 # Set BUCKET to the non-coldline Google Cloud Storage bucket
 export BUCKET=gs://$(gcloud config get-value project)/
 # Copy events.json into the bucket
-gsutil cp events.json ${BUCKET}
+gcloud storage cp events.json ${BUCKET}
diff --git a/quests/sparktobq/02_gcs.ipynb b/quests/sparktobq/02_gcs.ipynb
@@ -46,7 +46,7 @@
    "outputs": [],
    "source": [
     "BUCKET='cloud-training-demos-ml'  # CHANGE\n",
-    "!gsutil cp kdd* gs://$BUCKET/"
+    "!gcloud storage cp kdd* gs://$BUCKET/"
    ]
   },
   {
@@ -55,7 +55,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!gsutil ls gs://$BUCKET/kdd*"
+    "!gcloud storage ls gs://$BUCKET/kdd*"
    ]
   },
   {
@@ -189,8 +189,8 @@
    "outputs": [],
    "source": [
     "ax[0].get_figure().savefig('report.png');\n",
-    "!gsutil rm -rf gs://$BUCKET/sparktobq/\n",
-    "!gsutil cp report.png gs://$BUCKET/sparktobq/"
+    "!gcloud storage rm --recursive --continue-on-error gs://$BUCKET/sparktobq/\n",
+    "!gcloud storage cp report.png gs://$BUCKET/sparktobq/"
    ]
   },
   {
@@ -208,7 +208,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!gsutil ls gs://$BUCKET/sparktobq/**"
+    "!gcloud storage ls gs://$BUCKET/sparktobq/**"
    ]
   },
   {
diff --git a/quests/sparktobq/03_automate.ipynb b/quests/sparktobq/03_automate.ipynb
@@ -30,7 +30,7 @@
     "!wget http://kdd.ics.uci.edu/databases/kddcup99/kddcup.data_10_percent.gz\n",
     "BUCKET='cloud-training-demos-ml'  # CHANGE\n",
     "!pip install google-compute-engine\n",
-    "!gsutil cp kdd* gs://$BUCKET/"
+    "!gcloud storage cp kdd* gs://$BUCKET/"
    ]
   },
   {
@@ -40,7 +40,7 @@
    "outputs": [],
    "source": [
     "BUCKET='cloud-training-demos-ml'  # CHANGE\n",
-    "!gsutil ls gs://$BUCKET/kdd*"
+    "!gcloud storage ls gs://$BUCKET/kdd*"
    ]
   },
   {
@@ -193,8 +193,8 @@
     "%%writefile -a spark_analysis.py\n",
     "\n",
     "ax[0].get_figure().savefig('report.png');\n",
-    "#!gsutil rm -rf gs://$BUCKET/sparktobq/\n",
-    "#!gsutil cp report.png gs://$BUCKET/sparktobq/"
+    "#!gcloud storage rm --recursive --continue-on-error gs://$BUCKET/sparktobq/\n",
+    "#!gcloud storage cp report.png gs://$BUCKET/sparktobq/"
    ]
   },
   {
@@ -250,7 +250,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!gsutil ls gs://$BUCKET/sparktobq/**"
+    "!gcloud storage ls gs://$BUCKET/sparktobq/**"
    ]
   },
   {
diff --git a/quests/sparktobq/05_functions.ipynb b/quests/sparktobq/05_functions.ipynb
@@ -30,7 +30,7 @@
     "wget http://kdd.ics.uci.edu/databases/kddcup99/kddcup.data_10_percent.gz\n",
     "gunzip kddcup.data_10_percent.gz\n",
     "BUCKET='cloud-training-demos-ml'  # CHANGE\n",
-    "gsutil cp kdd* gs://$BUCKET/\n",
+    "gcloud storage cp kdd* gs://$BUCKET/\n",
     "bq mk sparktobq"
    ]
   },
@@ -234,8 +234,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!gsutil rm -rf gs://$BUCKET/sparktobq\n",
-    "!gsutil cp kddcup.data_10_percent gs://$BUCKET/"
+    "!gcloud storage rm --recursive --continue-on-error gs://$BUCKET/sparktobq\n",
+    "!gcloud storage cp kddcup.data_10_percent gs://$BUCKET/"
    ]
   },
   {
@@ -253,7 +253,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!gsutil ls gs://$BUCKET/sparktobq"
+    "!gcloud storage ls gs://$BUCKET/sparktobq"
    ]
   },
   {
diff --git a/quests/sparktobq/submit_workflow.sh b/quests/sparktobq/submit_workflow.sh
@@ -11,15 +11,15 @@ MACHINE_TYPE=n1-standard-4
 CLUSTER=sparktobq
 BUCKET=$1
 
-gsutil cp spark_analysis.py gs://$BUCKET/
+gcloud storage cp spark_analysis.py gs://$BUCKET/
 
 gcloud dataproc --quiet workflow-templates delete $TEMPLATE
 gcloud dataproc --quiet workflow-templates create $TEMPLATE
 
 # the things we need pip-installed on the cluster
 STARTUP_SCRIPT=gs://${BUCKET}/sparktobq/startup_script.sh
 echo "pip install --upgrade --quiet google-compute-engine google-cloud-storage matplotlib" > /tmp/startup_script.sh
-gsutil cp /tmp/startup_script.sh $STARTUP_SCRIPT
+gcloud storage cp /tmp/startup_script.sh $STARTUP_SCRIPT
 
 # create new cluster for job
 gcloud dataproc workflow-templates set-managed-cluster $TEMPLATE \

Original file line number	Diff line number	Diff line change
`@@ -46,7 +46,7 @@`
`46`	`46`	`"outputs": [],`
`47`	`47`	`"source": [`
`48`	`48`	`"BUCKET='cloud-training-demos-ml' # CHANGE\n",`
`49`		`- "!gsutil cp kdd* gs://$BUCKET/"`
	`49`	`+ "!gcloud storage cp kdd* gs://$BUCKET/"`
`50`	`50`	`]`
`51`	`51`	`},`
`52`	`52`	`{`
`@@ -55,7 +55,7 @@`
`55`	`55`	`"metadata": {},`
`56`	`56`	`"outputs": [],`
`57`	`57`	`"source": [`
`58`		`- "!gsutil ls gs://$BUCKET/kdd*"`
	`58`	`+ "!gcloud storage ls gs://$BUCKET/kdd*"`
`59`	`59`	`]`
`60`	`60`	`},`
`61`	`61`	`{`
`@@ -189,8 +189,8 @@`
`189`	`189`	`"outputs": [],`
`190`	`190`	`"source": [`
`191`	`191`	`"ax[0].get_figure().savefig('report.png');\n",`
`192`		`- "!gsutil rm -rf gs://$BUCKET/sparktobq/\n",`
`193`		`- "!gsutil cp report.png gs://$BUCKET/sparktobq/"`
	`192`	`+ "!gcloud storage rm --recursive --continue-on-error gs://$BUCKET/sparktobq/\n",`
	`193`	`+ "!gcloud storage cp report.png gs://$BUCKET/sparktobq/"`
`194`	`194`	`]`
`195`	`195`	`},`
`196`	`196`	`{`
`@@ -208,7 +208,7 @@`
`208`	`208`	`"metadata": {},`
`209`	`209`	`"outputs": [],`
`210`	`210`	`"source": [`
`211`		`- "!gsutil ls gs://$BUCKET/sparktobq/**"`
	`211`	`+ "!gcloud storage ls gs://$BUCKET/sparktobq/**"`
`212`	`212`	`]`
`213`	`213`	`},`
`214`	`214`	`{`
Original file line number	Diff line number	Diff line change
`@@ -30,7 +30,7 @@`
`30`	`30`	`"!wget http://kdd.ics.uci.edu/databases/kddcup99/kddcup.data_10_percent.gz\n",`
`31`	`31`	`"BUCKET='cloud-training-demos-ml' # CHANGE\n",`
`32`	`32`	`"!pip install google-compute-engine\n",`
`33`		`- "!gsutil cp kdd* gs://$BUCKET/"`
	`33`	`+ "!gcloud storage cp kdd* gs://$BUCKET/"`
`34`	`34`	`]`
`35`	`35`	`},`
`36`	`36`	`{`
`@@ -40,7 +40,7 @@`
`40`	`40`	`"outputs": [],`
`41`	`41`	`"source": [`
`42`	`42`	`"BUCKET='cloud-training-demos-ml' # CHANGE\n",`
`43`		`- "!gsutil ls gs://$BUCKET/kdd*"`
	`43`	`+ "!gcloud storage ls gs://$BUCKET/kdd*"`
`44`	`44`	`]`
`45`	`45`	`},`
`46`	`46`	`{`
`@@ -193,8 +193,8 @@`
`193`	`193`	`"%%writefile -a spark_analysis.py\n",`
`194`	`194`	`"\n",`
`195`	`195`	`"ax[0].get_figure().savefig('report.png');\n",`
`196`		`- "#!gsutil rm -rf gs://$BUCKET/sparktobq/\n",`
`197`		`- "#!gsutil cp report.png gs://$BUCKET/sparktobq/"`
	`196`	`+ "#!gcloud storage rm --recursive --continue-on-error gs://$BUCKET/sparktobq/\n",`
	`197`	`+ "#!gcloud storage cp report.png gs://$BUCKET/sparktobq/"`
`198`	`198`	`]`
`199`	`199`	`},`
`200`	`200`	`{`
`@@ -250,7 +250,7 @@`
`250`	`250`	`"metadata": {},`
`251`	`251`	`"outputs": [],`
`252`	`252`	`"source": [`
`253`		`- "!gsutil ls gs://$BUCKET/sparktobq/**"`
	`253`	`+ "!gcloud storage ls gs://$BUCKET/sparktobq/**"`
`254`	`254`	`]`
`255`	`255`	`},`
`256`	`256`	`{`