Merge pull request #3017 from googlyrahman/refactor-ai-migration-6ca0b3d1ec1a454ea32a74f32993f1c8

Migrate gsutil usage to gcloud storage

Author: jlpalomino

quests/data-science-on-gcp-edition1_tf2/03_sqlstudio/populate_table.sh

@@ -13,15 +13,14 @@ bash authorize_cloudshell.sh
 
 # the table name for mysqlimport comes from the filename, so rename our CSV files, changing bucket name as needed
 counter=0
-#for FILE in $(gsutil ls gs://${BUCKET}/flights/raw/2015*.csv); do
-#   gsutil cp $FILE flights.csv-${counter}
+#for FILE in $(gcloud storage ls gs://${BUCKET}/flights/raw/2015*.csv); do
+#   gcloud storage cp $FILE flights.csv-${counter}
 for FILE in 201501.csv 201507.csv; do
-   gsutil cp gs://${BUCKET}/flights/raw/$FILE flights.csv-${counter}
+   gcloud storage cp gs://${BUCKET}/flights/raw/$FILE flights.csv-${counter}
    counter=$((counter+1))
 done
 
 # import csv files
 MYSQLIP=$(gcloud sql instances describe flights --format="value(ipAddresses.ipAddress)")
 mysqlimport --local --host=$MYSQLIP --user=root --ignore-lines=1 --fields-terminated-by=',' --password bts flights.csv-*
 rm flights.csv-* 
-

quests/data-science-on-gcp-edition1_tf2/04_streaming/README.md

@@ -46,7 +46,7 @@ If you didn't go through Chapter 2, the simplest way to get the files you need i
 * In CloudShell, type:
 	```
 	bq mk flights
-	gsutil cp airports.csv.gz gs://<BUCKET-NAME>/flights/airports/airports.csv.gz
+	gcloud storage cp airports.csv.gz gs://<BUCKET-NAME>/flights/airports/airports.csv.gz
 	./df06.py -p $DEVSHELL_PROJECT_ID -b <BUCKETNAME> --region us-central1
 	``` 
 * Go to the GCP web console and wait for the Dataflow ch04timecorr job to finish. It might take several  
@@ -121,4 +121,4 @@ If you didn't go through Chapter 2, the simplest way to get the files you need i
 * Follow the steps in the chapter to connect to Data Studio and create a GeoMap.
 * Stop the simulation program in CloudShell.
 * From the GCP web console, stop the Dataflow streaming pipeline.
-	
+	

quests/data-science-on-gcp-edition1_tf2/04_streaming/ingest_from_crsbucket.sh

@@ -10,7 +10,7 @@ FROM=gs://data-science-on-gcp/flights/tzcorr
 TO=gs://$BUCKET/flights/tzcorr
 
 #sharded files
-CMD="gsutil -m cp "
+CMD="gcloud storage cp "
 for SHARD in `seq -w 0 35`; do
   CMD="$CMD ${FROM}/all_flights-000${SHARD}-of-00036"
 done
@@ -19,7 +19,7 @@ echo $CMD
 $CMD
 
 # individual files
-CMD="gsutil -m cp "
+CMD="gcloud storage cp "
 for FILE in airports/airports.csv.gz; do
    CMD="$CMD gs://data-science-on-gcp/flights/$FILE gs://$BUCKET/flights/$FILE"
 done

quests/data-science-on-gcp-edition1_tf2/04_streaming/simulate/get_sminput.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-gsutil cat gs://cloud-training-demos-ml/flights/raw/201501.csv | head -1000 > 201501_part.csv
+gcloud storage cat gs://cloud-training-demos-ml/flights/raw/201501.csv | head -1000 > 201501_part.csv

quests/data-science-on-gcp-edition1_tf2/05_bqnotebook/load_into_bq.sh

@@ -9,7 +9,7 @@ BUCKET=$1
 
 bq show --format=prettyjson flights.simevents > simevents.json
 
-ONE_FILE=$(gsutil ls gs://${BUCKET}/flights/tzcorr/all_flights-00001*)
+ONE_FILE=$(gcloud storage ls gs://${BUCKET}/flights/tzcorr/all_flights-00001*)
 echo "Creating table definition from $ONE_FILE"
 bq mk --external_table_definition=./tzcorr.json@CSV=$ONE_FILE flights.fedtzcorr
 

quests/data-science-on-gcp-edition1_tf2/09_cloudml/deploy_model.sh

@@ -16,7 +16,7 @@ REGION=$3
 
 MODEL_NAME=flights
 VERSION_NAME=tf2
-EXPORT_PATH=$(gsutil ls gs://$BUCKET/flights/trained_model/${BEST_MODEL}export | tail -1)
+EXPORT_PATH=$(gcloud storage ls gs://$BUCKET/flights/trained_model/${BEST_MODEL}export | tail -1)
 echo $EXPORT_PATH
 
 if [[ $(gcloud ai-platform models list --format='value(name)' --region=$REGION | grep $MODEL_NAME) ]]; then

quests/data-science-on-gcp-edition1_tf2/09_cloudml/flights_model_tf2.ipynb

@@ -133,7 +133,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!gsutil ls $DATA_BUCKET"
+    "!gcloud storage ls $DATA_BUCKET"
    ]
   },
   {
@@ -329,7 +329,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!gsutil -m rm -rf $OUTDIR"
+    "!gcloud storage rm --recursive --continue-on-error $OUTDIR"
    ]
   },
   {
@@ -457,7 +457,7 @@
    "outputs": [],
    "source": [
     "%%bash\n",
-    "model_dir=$(gsutil ls ${OUTDIR}/export | tail -1)\n",
+    "model_dir=$(gcloud storage ls ${OUTDIR}/export | tail -1)\n",
     "echo $model_dir\n",
     "saved_model_cli show --tag_set serve --signature_def serving_default --dir $model_dir"
    ]
@@ -474,7 +474,7 @@
     "REGION=us-east1\n",
     "MODEL_NAME=flights\n",
     "VERSION_NAME=tf2\n",
-    "EXPORT_PATH=$(gsutil ls ${OUTDIR}/export | tail -1)\n",
+    "EXPORT_PATH=$(gcloud storage ls ${OUTDIR}/export | tail -1)\n",
     "\n",
     "if [[ $(gcloud ai-platform models list --format='value(name)' | grep $MODEL_NAME) ]]; then\n",
     "    echo \"$MODEL_NAME already exists\"\n",

quests/data-science-on-gcp-edition1_tf2/10_realtime/eval.sh

@@ -8,11 +8,10 @@ fi
 BUCKET=$1
 PROJECT=$2
 
-gsutil -m rm -rf gs://$BUCKET/flights/chapter10/eval
+gcloud storage rm --recursive --continue-on-error gs://$BUCKET/flights/chapter10/eval
 
 cd chapter10
 
 mvn compile exec:java \
  -Dexec.mainClass=com.google.cloud.training.flights.EvaluateModel \
  -Dexec.args="--fullDataset --maxNumWorkers=10 --autoscalingAlgorithm=THROUGHPUT_BASED --bucket=$BUCKET --project=$PROJECT"
-

quests/data-science-on-gcp-edition1_tf2/10_realtime/ingest_2016.sh

@@ -11,5 +11,5 @@ export YEAR=2016
 bash ../02_ingest/download.sh
 bash ../02_ingest/zip_to_csv.sh
 bash ../02_ingest/quotes_comma.sh
-gsutil -m cp *.csv gs://$BUCKET/flights2016/raw
+gcloud storage cp *.csv gs://$BUCKET/flights2016/raw
 rm *.csv

quests/data-science-on-gcp-edition1_tf2/10_realtime/predict.sh

@@ -8,12 +8,11 @@ fi
 BUCKET=$1
 PROJECT=$2
 
-gsutil -m rm -rf gs://$BUCKET/flights/chapter10/output
+gcloud storage rm --recursive --continue-on-error gs://$BUCKET/flights/chapter10/output
 bq rm -f flights.predictions
 
 cd chapter10
 
 mvn compile exec:java \
  -Dexec.mainClass=com.google.cloud.training.flights.AddRealtimePrediction \
  -Dexec.args="--realtime --speedupFactor=60 --maxNumWorkers=10 --autoscalingAlgorithm=THROUGHPUT_BASED --bucket=$BUCKET --project=$PROJECT"
-