Merge pull request #3012 from googlyrahman/refactor-ai-migration-e8ef66d3acc241e3888d3424275d2c4d

Migrate gsutil usage to gcloud storage

Author: WorkingGitHubUser

courses/data_analysis/deepdive/README.md

@@ -1,7 +1,6 @@
 
-
 1. Download data:
-gsutil -m cp gs://cloud-training-demos/datasme/pubsub/actions.csv pubsub-exercises/data/actions.csv
+gcloud storage cp gs://cloud-training-demos/datasme/pubsub/actions.csv pubsub-exercises/data/actions.csv
 
 2. PubSub exercises:
 https://codelabs.developers.google.com/codelabs/datasme-pubsub-01

courses/data_analysis/deepdive/bigtable-exercises/download_data.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-gsutil -m cp gs://cloud-bigtable-training/actions_subset.csv src/main/resources/actions_subset.csv
+gcloud storage cp gs://cloud-bigtable-training/actions_subset.csv src/main/resources/actions_subset.csv

courses/data_analysis/deepdive/composer-exercises/hello_world_solution.py

@@ -30,7 +30,7 @@
   """
 
   # Solution:
-  # gsutil cp count.txt gs://{your-composer-bucket}/dags
+  # gcloud storage cp count.txt gs://{your-composer-bucket}/dags
   with open('/home/airflow/gcs/dags/count.txt') as f:
     number_of_templated_tasks = int(f.read())
     for i in xrange(number_of_templated_tasks):

courses/data_analysis/deepdive/pubsub-exercises/download_data.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-gsutil -m cp gs://cloud-training-demos/datasme/pubsub/actions.csv data/actions.csv
+gcloud storage cp gs://cloud-training-demos/datasme/pubsub/actions.csv data/actions.csv

quests/sparktobq/04_bigquery.ipynb

@@ -29,7 +29,7 @@
     "# Catch-up cell. Run if you did not do previous notebooks of this sequence\n",
     "!wget http://kdd.ics.uci.edu/databases/kddcup99/kddcup.data_10_percent.gz\n",
     "BUCKET='cloud-training-demos-ml'  # CHANGE\n",
-    "!gsutil cp kdd* gs://$BUCKET/"
+    "!gcloud storage cp kdd* gs://$BUCKET/"
    ]
   },
   {

quests/sparktobq/05_functions_dw.ipynb

@@ -30,7 +30,7 @@
     "wget http://kdd.ics.uci.edu/databases/kddcup99/kddcup.data_10_percent.gz\n",
     "gunzip kddcup.data_10_percent.gz\n",
     "BUCKET='cloud-training-demos-ml'  # CHANGE\n",
-    "gsutil cp kdd* gs://$BUCKET/\n",
+    "gcloud storage cp kdd* gs://$BUCKET/\n",
     "bq mk sparktobq"
    ]
   },
@@ -234,7 +234,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!gsutil cp kddcup.data_10_percent gs://$BUCKET/"
+    "!gcloud storage cp kddcup.data_10_percent gs://$BUCKET/"
    ]
   },
   {
@@ -252,7 +252,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!gsutil ls gs://$BUCKET/sparktobq"
+    "!gcloud storage ls gs://$BUCKET/sparktobq"
    ]
   },
   {

quests/sparktobq/README.md

@@ -55,4 +55,3 @@
 ### Part 5
 * On the same AI Platform Notebooks instance as Part 4, open and run ```05_functions.ipynb```
 * Delete the Notebooks instance
-

quests/sparktobq/copy_from_gcs.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 BUCKET=cloud-training-demos-ml  # CHANGE
 
-gsutil cp gs://$BUCKET/notebooks/jupyter/* .
+gcloud storage cp gs://$BUCKET/notebooks/jupyter/* .

quests/sparktobq/copy_to_gcs.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 BUCKET=cloud-training-demos-ml  # CHANGE
 
-gsutil cp *.ipynb gs://$BUCKET/notebooks/jupyter
+gcloud storage cp *.ipynb gs://$BUCKET/notebooks/jupyter

quests/sparktobq/spark_analysis.py

@@ -1,4 +1,3 @@
-
 import argparse
 parser = argparse.ArgumentParser()
 parser.add_argument("--bucket", help="bucket for input and output")
@@ -60,8 +59,8 @@
 attack_stats.show()
 ax = attack_stats.toPandas().plot.bar(x='protocol_type', subplots=True, figsize=(10,25))
 ax[0].get_figure().savefig('report.png');
-#!gsutil rm -rf gs://$BUCKET/sparktobq/
-#!gsutil cp report.png gs://$BUCKET/sparktobq/
+#!gcloud storage rm --recursive --continue-on-error gs://$BUCKET/sparktobq/
+#!gcloud storage cp report.png gs://$BUCKET/sparktobq/
 import google.cloud.storage as gcs
 bucket = gcs.Client().get_bucket(BUCKET)
 for blob in bucket.list_blobs(prefix='sparktobq/'):