diff --git a/blogs/batch_normalization/estimator_batch_normalization.ipynb b/blogs/batch_normalization/estimator_batch_normalization.ipynb index bace864089..834f7a5d58 100644 --- a/blogs/batch_normalization/estimator_batch_normalization.ipynb +++ b/blogs/batch_normalization/estimator_batch_normalization.ipynb @@ -91,7 +91,7 @@ " JOBNAME=mnist$netname_$(date -u +%y%m%d_%H%M%S)\n", " OUTDIR=gs://${BUCKET}/mnist_models/mnist_model$netname/trained_model\n", " echo $OUTDIR $REGION $JOBNAME\n", - " gsutil -m rm -rf $OUTDIR\n", + " gcloud storage rm --recursive --continue-on-error $OUTDIR\n", " submitMLEngineJob\n", " net=$net, \n", " done \n", diff --git a/blogs/explainable_ai/AI_Explanations_on_CAIP.ipynb b/blogs/explainable_ai/AI_Explanations_on_CAIP.ipynb index f5d24a9de9..754c47f8ea 100644 --- a/blogs/explainable_ai/AI_Explanations_on_CAIP.ipynb +++ b/blogs/explainable_ai/AI_Explanations_on_CAIP.ipynb @@ -256,17 +256,14 @@ }, "source": [ "%%bash\n", - "exists=$(gsutil ls -d | grep -w gs://${BUCKET_NAME}/)\n", - "\n", + "exists=$(gcloud storage ls | grep -w gs://${BUCKET_NAME}/)\n", "\n", "if [ -n \"$exists\" ]; then\n", " echo -e \"Bucket gs://${BUCKET_NAME} already exists.\"\n", " \n", "else\n", " echo \"Creating a new GCS bucket.\"\n", - " gsutil mb -l ${REGION} gs://${BUCKET_NAME}\n", - " echo -e \"\\nHere are your current buckets:\"\n", - " gsutil ls\n", - "fi" + " gcloud storage buckets create --location=${REGION} gs://${BUCKET_NAME}\n", " echo -e \"\\nHere are your current buckets:\"\n", + " gcloud storage ls\n", "fi" ], "execution_count": 0, "outputs": [] @@ -330,8 +327,7 @@ "%%bash\n", "# Copy the data to your notebook instance\n", "mkdir taxi_preproc\n", - "gsutil cp -r gs://cloud-training/bootcamps/serverlessml/taxi_preproc/*_xai.csv ./taxi_preproc\n", - "ls -l taxi_preproc" + "gcloud storage cp --recursive gs://cloud-training/bootcamps/serverlessml/taxi_preproc/*_xai.csv ./taxi_preproc\n", "ls -l taxi_preproc" ], "execution_count": 0, "outputs": [] @@ -812,8 +808,7 @@ "colab": {} }, "source": [ - "!gsutil cp explanation_metadata.json $export_path" - ], + "!gcloud storage cp explanation_metadata.json $export_path" ], "execution_count": 0, "outputs": [] }, @@ -1107,4 +1102,4 @@ "outputs": [] } ] -} \ No newline at end of file +} diff --git a/blogs/gcp_forecasting/gcp_time_series_forecasting.ipynb b/blogs/gcp_forecasting/gcp_time_series_forecasting.ipynb index 8420d82424..4ba2c89266 100644 --- a/blogs/gcp_forecasting/gcp_time_series_forecasting.ipynb +++ b/blogs/gcp_forecasting/gcp_time_series_forecasting.ipynb @@ -1488,8 +1488,7 @@ }, "outputs": [], "source": [ - "!gsutil -m cp gs://asl-testing-bucket/forecasting/nyc_real_estate/data/*.csv ." - ] +"!gcloud storage cp gs://asl-testing-bucket/forecasting/nyc_real_estate/data/*.csv ." ] }, { "cell_type": "code", @@ -1551,8 +1550,7 @@ "OUTDIR=gs://$BUCKET/forecasting/nyc_real_estate/trained_model\n", "JOBNAME=nyc_real_estate$(date -u +%y%m%d_%H%M%S)\n", "echo $OUTDIR $REGION $JOBNAME\n", - "gsutil -m rm -rf $OUTDIR\n", - "gcloud ai-platform jobs submit training $JOBNAME \\\n", + "gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ai-platform jobs submit training $JOBNAME \\\n", " --region=$REGION \\\n", " --module-name=trainer.task \\\n", " --package-path=$PWD/tf_module/trainer \\\n", diff --git a/blogs/lightning/1_explore.ipynb b/blogs/lightning/1_explore.ipynb index 4e2a8b2710..f87e402b2c 100644 --- a/blogs/lightning/1_explore.ipynb +++ b/blogs/lightning/1_explore.ipynb @@ -1142,8 +1142,7 @@ "LABEL_RADIUS=2\n", "STRIDE=4 # use 2*label_patch_radius\n", "OUTDIR=gs://${BUCKET}/lightning/preproc_${LATLONRES}_${TRAIN_RADIUS}_${LABEL_RADIUS}\n", - "gsutil -m rm -rf $OUTDIR\n", - "\n", + "gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "\n", "python3 -m ltgpred.preproc.create_dataset \\\n", " --outdir=$OUTDIR \\\n", " --startday 2018-45 --endday 2019-111 --project=$PROJECT \\\n", @@ -1172,8 +1171,7 @@ "STRIDE=2 # use 2*label_patch_radius\n", "LTG_VALID_TIME=5\n", "OUTDIR=gs://${BUCKET}/lightning/preproc_${LATLONRES}_${TRAIN_RADIUS}_${LABEL_RADIUS}\n", - "gsutil -m rm -rf $OUTDIR\n", - "\n", + "gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "\n", "python3 -m ltgpred.preproc.create_dataset \\\n", " --outdir=$OUTDIR \\\n", " --startday 2018-45 --endday 2019-111 --project=$PROJECT \\\n", @@ -1197,8 +1195,7 @@ ], "source": [ "%%bash\n", - "gsutil cat gs://cloud-training-demos-ml/lightning/preproc_0.02_32_1/stats*" - ] + "gcloud storage cat gs://cloud-training-demos-ml/lightning/preproc_0.02_32_1/stats*" ] }, { "cell_type": "markdown", diff --git a/blogs/lightning/2_sklearn.ipynb b/blogs/lightning/2_sklearn.ipynb index 279605ed0e..b9620c4f5c 100644 --- a/blogs/lightning/2_sklearn.ipynb +++ b/blogs/lightning/2_sklearn.ipynb @@ -108,7 +108,7 @@ ], "source": [ "!mkdir -p preproc/csv\n", - "!gsutil cp gs://$BUCKET/lightning/preproc_0.02_32_2/csv/*-00000-* preproc/csv" + "!gcloud storage cp gs://$BUCKET/lightning/preproc_0.02_32_2/csv/*-00000-* preproc/csv" ] }, { @@ -336,7 +336,7 @@ "OUTDIR=gs://${BUCKET}/lightning/skl_trained\n", "DATADIR=gs://$BUCKET/lightning/preproc_0.02_32_2/csv\n", "JOBNAME=ltgpred_skl_$(date -u +%y%m%d_%H%M%S)\n", - "gsutil -m rm -rf $OUTDIR\n", + "gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n", " --module-name=ltgpred.trainer.train_skl --package-path=${PWD}/ltgpred --job-dir=$OUTDIR \\\n", " --region=${REGION} --scale-tier=custom --config=largemachine.yaml \\\n", diff --git a/blogs/lightning/3_convnet.ipynb b/blogs/lightning/3_convnet.ipynb index a78f63f16b..b09208d204 100644 --- a/blogs/lightning/3_convnet.ipynb +++ b/blogs/lightning/3_convnet.ipynb @@ -125,7 +125,7 @@ ], "source": [ "!mkdir -p preproc/tfrecord\n", - "!gsutil cp gs://$BUCKET/lightning/preproc_0.02_32_2/tfrecord/*-00000-* preproc/tfrecord" + "!gcloud storage cp gs://$BUCKET/lightning/preproc_0.02_32_2/tfrecord/*-00000-* preproc/tfrecord" ] }, { @@ -275,7 +275,7 @@ "for ARCH in convnet; do\n", " JOBNAME=ltgpred_${ARCH}_$(date -u +%y%m%d_%H%M%S)\n", " OUTDIR=gs://${BUCKET}/lightning/${ARCH}_trained_gpu\n", - " gsutil -m rm -rf $OUTDIR\n", + " gcloud storage rm --recursive --continue-on-error $OUTDIR\n", " gcloud ml-engine jobs submit training $JOBNAME \\\n", " --module-name trainer.train_cnn --package-path ${PWD}/ltgpred/trainer --job-dir=$OUTDIR \\\n", " --region=${REGION} --scale-tier CUSTOM --config largemachine.yaml \\\n", @@ -383,7 +383,7 @@ "OUTDIR=gs://${BUCKET}/lightning/convnet_trained_gpu_hparam\n", "DATADIR=gs://$BUCKET/lightning/preproc_0.02_32_2/tfrecord\n", "JOBNAME=ltgpred_hparam_$(date -u +%y%m%d_%H%M%S)\n", - "gsutil -m rm -rf $OUTDIR\n", + "gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n", " --module-name=ltgpred.trainer.train_cnn --package-path=${PWD}/ltgpred --job-dir=$OUTDIR \\\n", " --region=${REGION} --scale-tier=CUSTOM --config=hyperparam_gpu.yaml \\\n", @@ -440,7 +440,7 @@ "OUTDIR=gs://${BUCKET}/lightning/cnn_trained_tpu\n", "DATADIR=gs://$BUCKET/lightning/preproc_0.02_32_2/tfrecord\n", "JOBNAME=ltgpred_cnn_$(date -u +%y%m%d_%H%M%S)\n", - "gsutil -m rm -rf $OUTDIR\n", + "gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n", " --module-name ltgpred.trainer.train_cnn --package-path ${PWD}/ltgpred --job-dir=$OUTDIR \\\n", " --region ${REGION} --scale-tier BASIC_TPU \\\n", @@ -481,7 +481,7 @@ "for ARCH in feateng; do\n", " JOBNAME=ltgpred_${ARCH}_$(date -u +%y%m%d_%H%M%S)\n", " OUTDIR=gs://${BUCKET}/lightning/${ARCH}_trained_gpu\n", - " gsutil -m rm -rf $OUTDIR\n", + " gcloud storage rm --recursive --continue-on-error $OUTDIR\n", " gcloud ml-engine jobs submit training $JOBNAME \\\n", " --module-name ltgpred.trainer.train_cnn --package-path ${PWD}/ltgpred --job-dir=$OUTDIR \\\n", " --region=${REGION} --scale-tier CUSTOM --config largemachine.yaml \\\n", diff --git a/blogs/lightning/README.md b/blogs/lightning/README.md index 3dfa5e6a3e..78e4ef4ae1 100644 --- a/blogs/lightning/README.md +++ b/blogs/lightning/README.md @@ -30,7 +30,7 @@ LABEL_RADIUS=1 STRIDE=2 # use 2*label_patch_radius LTG_VALID_TIME=5 OUTDIR=gs://${BUCKET}/lightning/preproc_${LATLONRES}_${TRAIN_RADIUS}_${LABEL_RADIUS} -gsutil -m rm -rf $OUTDIR +gcloud storage rm --recursive --continue-on-error $OUTDIR python -m ltgpred.preproc.create_dataset \ --outdir=$OUTDIR \ @@ -52,7 +52,7 @@ DATADIR=gs://${BUCKET}/lightning/preproc_${LATLONRES}_${TRAIN_RADIUS}_${LABEL_RA for ARCH in convnet; do JOBNAME=ltgpred_${ARCH}_$(date -u +%y%m%d_%H%M%S) OUTDIR=gs://${BUCKET}/lightning/${ARCH}_trained_gpu - gsutil -m rm -rf $OUTDIR + gcloud storage rm --recursive --continue-on-error $OUTDIR gcloud ml-engine jobs submit training $JOBNAME \ --module-name trainer.train_cnn --package-path ${PWD}/ltgpred/trainer --job-dir=$OUTDIR \ --region=${REGION} --scale-tier CUSTOM --config largemachine.yaml \ diff --git a/blogs/lightning/ltgpred/trainer/train_skl.py b/blogs/lightning/ltgpred/trainer/train_skl.py index db7c13cce8..e02ecfd13b 100644 --- a/blogs/lightning/ltgpred/trainer/train_skl.py +++ b/blogs/lightning/ltgpred/trainer/train_skl.py @@ -107,7 +107,7 @@ def save_model(model_tosave, gcspath, name): # pylint: disable=unused-argument model_path = os.path.join( gcspath, datetime.datetime.now().strftime('export_%Y%m%d_%H%M%S'), filename) - subprocess.check_call(['gsutil', 'cp', filename, model_path]) + subprocess.check_call(['gcloud', 'storage', 'cp', filename, model_path]) return model_path diff --git a/blogs/sklearn/babyweight/trainer/model.py b/blogs/sklearn/babyweight/trainer/model.py index e18c1203e6..779c0d46c6 100644 --- a/blogs/sklearn/babyweight/trainer/model.py +++ b/blogs/sklearn/babyweight/trainer/model.py @@ -138,5 +138,5 @@ def save_model(estimator, gcspath, name): joblib.dump(estimator, model) model_path = os.path.join(gcspath, datetime.datetime.now().strftime( 'export_%Y%m%d_%H%M%S'), model) - subprocess.check_call(['gsutil', 'cp', model, model_path]) + subprocess.check_call(['gcloud', 'storage', 'cp', model, model_path]) return model_path diff --git a/blogs/sklearn/babyweight_skl.ipynb b/blogs/sklearn/babyweight_skl.ipynb index b0b88f5649..0234f46e2a 100644 --- a/blogs/sklearn/babyweight_skl.ipynb +++ b/blogs/sklearn/babyweight_skl.ipynb @@ -103,9 +103,7 @@ "outputs": [], "source": [ "%%bash\n", - "if ! gsutil ls | grep -q gs://${BUCKET}/; then\n", - " gsutil mb -l ${REGION} gs://${BUCKET}\n", - "fi" + "if ! gcloud storage ls | grep -q gs://${BUCKET}/; then\n", " gcloud storage buckets create --location ${REGION} gs://${BUCKET}\n", "fi" ] }, { @@ -840,8 +838,7 @@ " joblib.dump(estimator, model)\n", " model_path = os.path.join(gcspath, datetime.datetime.now().strftime(\n", " 'export_%Y%m%d_%H%M%S'), model)\n", - " subprocess.check_call(['gsutil', 'cp', model, model_path])\n", - " return model_path" + " subprocess.check_call(['gcloud', 'storage', 'cp', model, model_path])\n", " return model_path" ] }, { @@ -1199,8 +1196,7 @@ ], "source": [ "%bash\n", - "gsutil ls gs://${BUCKET}/babyweight/sklearn/ | tail -1" - ] + "gcloud storage ls gs://${BUCKET}/babyweight/sklearn/ | tail -1" ] }, { "cell_type": "code", @@ -1218,8 +1214,7 @@ "%bash\n", "MODEL_NAME=\"babyweight\"\n", "MODEL_VERSION=\"skl\"\n", - "MODEL_LOCATION=$(gsutil ls gs://${BUCKET}/babyweight/sklearn/ | tail -1)\n", - "echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n", + "MODEL_LOCATION=$(gcloud storage ls gs://${BUCKET}/babyweight/sklearn/ | tail -1)\n", "echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n", "#gcloud ml-engine versions delete ${MODEL_VERSION} --model ${MODEL_NAME}\n", "#gcloud ml-engine models delete ${MODEL_NAME}\n", "#gcloud ml-engine models create ${MODEL_NAME} --regions $REGION\n",