Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@
" JOBNAME=mnist$netname_$(date -u +%y%m%d_%H%M%S)\n",
" OUTDIR=gs://${BUCKET}/mnist_models/mnist_model$netname/trained_model\n",
" echo $OUTDIR $REGION $JOBNAME\n",
" gsutil -m rm -rf $OUTDIR\n",
" gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
" submitMLEngineJob\n",
" net=$net, \n",
" done \n",
Expand Down
17 changes: 6 additions & 11 deletions blogs/explainable_ai/AI_Explanations_on_CAIP.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -256,17 +256,14 @@
},
"source": [
"%%bash\n",
"exists=$(gsutil ls -d | grep -w gs://${BUCKET_NAME}/)\n",
"\n",
"exists=$(gcloud storage ls | grep -w gs://${BUCKET_NAME}/)\n", "\n",
"if [ -n \"$exists\" ]; then\n",
" echo -e \"Bucket gs://${BUCKET_NAME} already exists.\"\n",
" \n",
"else\n",
" echo \"Creating a new GCS bucket.\"\n",
" gsutil mb -l ${REGION} gs://${BUCKET_NAME}\n",
" echo -e \"\\nHere are your current buckets:\"\n",
" gsutil ls\n",
"fi"
" gcloud storage buckets create --location=${REGION} gs://${BUCKET_NAME}\n", " echo -e \"\\nHere are your current buckets:\"\n",
" gcloud storage ls\n", "fi"
],
"execution_count": 0,
"outputs": []
Expand Down Expand Up @@ -330,8 +327,7 @@
"%%bash\n",
"# Copy the data to your notebook instance\n",
"mkdir taxi_preproc\n",
"gsutil cp -r gs://cloud-training/bootcamps/serverlessml/taxi_preproc/*_xai.csv ./taxi_preproc\n",
"ls -l taxi_preproc"
"gcloud storage cp --recursive gs://cloud-training/bootcamps/serverlessml/taxi_preproc/*_xai.csv ./taxi_preproc\n", "ls -l taxi_preproc"
],
"execution_count": 0,
"outputs": []
Expand Down Expand Up @@ -812,8 +808,7 @@
"colab": {}
},
"source": [
"!gsutil cp explanation_metadata.json $export_path"
],
"!gcloud storage cp explanation_metadata.json $export_path" ],
"execution_count": 0,
"outputs": []
},
Expand Down Expand Up @@ -1107,4 +1102,4 @@
"outputs": []
}
]
}
}
6 changes: 2 additions & 4 deletions blogs/gcp_forecasting/gcp_time_series_forecasting.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -1488,8 +1488,7 @@
},
"outputs": [],
"source": [
"!gsutil -m cp gs://asl-testing-bucket/forecasting/nyc_real_estate/data/*.csv ."
]
"!gcloud storage cp gs://asl-testing-bucket/forecasting/nyc_real_estate/data/*.csv ." ]
},
{
"cell_type": "code",
Expand Down Expand Up @@ -1551,8 +1550,7 @@
"OUTDIR=gs://$BUCKET/forecasting/nyc_real_estate/trained_model\n",
"JOBNAME=nyc_real_estate$(date -u +%y%m%d_%H%M%S)\n",
"echo $OUTDIR $REGION $JOBNAME\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud ai-platform jobs submit training $JOBNAME \\\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ai-platform jobs submit training $JOBNAME \\\n",
" --region=$REGION \\\n",
" --module-name=trainer.task \\\n",
" --package-path=$PWD/tf_module/trainer \\\n",
Expand Down
9 changes: 3 additions & 6 deletions blogs/lightning/1_explore.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -1142,8 +1142,7 @@
"LABEL_RADIUS=2\n",
"STRIDE=4 # use 2*label_patch_radius\n",
"OUTDIR=gs://${BUCKET}/lightning/preproc_${LATLONRES}_${TRAIN_RADIUS}_${LABEL_RADIUS}\n",
"gsutil -m rm -rf $OUTDIR\n",
"\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "\n",
"python3 -m ltgpred.preproc.create_dataset \\\n",
" --outdir=$OUTDIR \\\n",
" --startday 2018-45 --endday 2019-111 --project=$PROJECT \\\n",
Expand Down Expand Up @@ -1172,8 +1171,7 @@
"STRIDE=2 # use 2*label_patch_radius\n",
"LTG_VALID_TIME=5\n",
"OUTDIR=gs://${BUCKET}/lightning/preproc_${LATLONRES}_${TRAIN_RADIUS}_${LABEL_RADIUS}\n",
"gsutil -m rm -rf $OUTDIR\n",
"\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "\n",
"python3 -m ltgpred.preproc.create_dataset \\\n",
" --outdir=$OUTDIR \\\n",
" --startday 2018-45 --endday 2019-111 --project=$PROJECT \\\n",
Expand All @@ -1197,8 +1195,7 @@
],
"source": [
"%%bash\n",
"gsutil cat gs://cloud-training-demos-ml/lightning/preproc_0.02_32_1/stats*"
]
"gcloud storage cat gs://cloud-training-demos-ml/lightning/preproc_0.02_32_1/stats*" ]
},
{
"cell_type": "markdown",
Expand Down
4 changes: 2 additions & 2 deletions blogs/lightning/2_sklearn.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@
],
"source": [
"!mkdir -p preproc/csv\n",
"!gsutil cp gs://$BUCKET/lightning/preproc_0.02_32_2/csv/*-00000-* preproc/csv"
"!gcloud storage cp gs://$BUCKET/lightning/preproc_0.02_32_2/csv/*-00000-* preproc/csv"
]
},
{
Expand Down Expand Up @@ -336,7 +336,7 @@
"OUTDIR=gs://${BUCKET}/lightning/skl_trained\n",
"DATADIR=gs://$BUCKET/lightning/preproc_0.02_32_2/csv\n",
"JOBNAME=ltgpred_skl_$(date -u +%y%m%d_%H%M%S)\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --module-name=ltgpred.trainer.train_skl --package-path=${PWD}/ltgpred --job-dir=$OUTDIR \\\n",
" --region=${REGION} --scale-tier=custom --config=largemachine.yaml \\\n",
Expand Down
10 changes: 5 additions & 5 deletions blogs/lightning/3_convnet.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@
],
"source": [
"!mkdir -p preproc/tfrecord\n",
"!gsutil cp gs://$BUCKET/lightning/preproc_0.02_32_2/tfrecord/*-00000-* preproc/tfrecord"
"!gcloud storage cp gs://$BUCKET/lightning/preproc_0.02_32_2/tfrecord/*-00000-* preproc/tfrecord"
]
},
{
Expand Down Expand Up @@ -275,7 +275,7 @@
"for ARCH in convnet; do\n",
" JOBNAME=ltgpred_${ARCH}_$(date -u +%y%m%d_%H%M%S)\n",
" OUTDIR=gs://${BUCKET}/lightning/${ARCH}_trained_gpu\n",
" gsutil -m rm -rf $OUTDIR\n",
" gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
" gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --module-name trainer.train_cnn --package-path ${PWD}/ltgpred/trainer --job-dir=$OUTDIR \\\n",
" --region=${REGION} --scale-tier CUSTOM --config largemachine.yaml \\\n",
Expand Down Expand Up @@ -383,7 +383,7 @@
"OUTDIR=gs://${BUCKET}/lightning/convnet_trained_gpu_hparam\n",
"DATADIR=gs://$BUCKET/lightning/preproc_0.02_32_2/tfrecord\n",
"JOBNAME=ltgpred_hparam_$(date -u +%y%m%d_%H%M%S)\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --module-name=ltgpred.trainer.train_cnn --package-path=${PWD}/ltgpred --job-dir=$OUTDIR \\\n",
" --region=${REGION} --scale-tier=CUSTOM --config=hyperparam_gpu.yaml \\\n",
Expand Down Expand Up @@ -440,7 +440,7 @@
"OUTDIR=gs://${BUCKET}/lightning/cnn_trained_tpu\n",
"DATADIR=gs://$BUCKET/lightning/preproc_0.02_32_2/tfrecord\n",
"JOBNAME=ltgpred_cnn_$(date -u +%y%m%d_%H%M%S)\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --module-name ltgpred.trainer.train_cnn --package-path ${PWD}/ltgpred --job-dir=$OUTDIR \\\n",
" --region ${REGION} --scale-tier BASIC_TPU \\\n",
Expand Down Expand Up @@ -481,7 +481,7 @@
"for ARCH in feateng; do\n",
" JOBNAME=ltgpred_${ARCH}_$(date -u +%y%m%d_%H%M%S)\n",
" OUTDIR=gs://${BUCKET}/lightning/${ARCH}_trained_gpu\n",
" gsutil -m rm -rf $OUTDIR\n",
" gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
" gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --module-name ltgpred.trainer.train_cnn --package-path ${PWD}/ltgpred --job-dir=$OUTDIR \\\n",
" --region=${REGION} --scale-tier CUSTOM --config largemachine.yaml \\\n",
Expand Down
4 changes: 2 additions & 2 deletions blogs/lightning/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ LABEL_RADIUS=1
STRIDE=2 # use 2*label_patch_radius
LTG_VALID_TIME=5
OUTDIR=gs://${BUCKET}/lightning/preproc_${LATLONRES}_${TRAIN_RADIUS}_${LABEL_RADIUS}
gsutil -m rm -rf $OUTDIR
gcloud storage rm --recursive --continue-on-error $OUTDIR

python -m ltgpred.preproc.create_dataset \
--outdir=$OUTDIR \
Expand All @@ -52,7 +52,7 @@ DATADIR=gs://${BUCKET}/lightning/preproc_${LATLONRES}_${TRAIN_RADIUS}_${LABEL_RA
for ARCH in convnet; do
JOBNAME=ltgpred_${ARCH}_$(date -u +%y%m%d_%H%M%S)
OUTDIR=gs://${BUCKET}/lightning/${ARCH}_trained_gpu
gsutil -m rm -rf $OUTDIR
gcloud storage rm --recursive --continue-on-error $OUTDIR
gcloud ml-engine jobs submit training $JOBNAME \
--module-name trainer.train_cnn --package-path ${PWD}/ltgpred/trainer --job-dir=$OUTDIR \
--region=${REGION} --scale-tier CUSTOM --config largemachine.yaml \
Expand Down
2 changes: 1 addition & 1 deletion blogs/lightning/ltgpred/trainer/train_skl.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def save_model(model_tosave, gcspath, name): # pylint: disable=unused-argument
model_path = os.path.join(
gcspath,
datetime.datetime.now().strftime('export_%Y%m%d_%H%M%S'), filename)
subprocess.check_call(['gsutil', 'cp', filename, model_path])
subprocess.check_call(['gcloud', 'storage', 'cp', filename, model_path])
return model_path


Expand Down
2 changes: 1 addition & 1 deletion blogs/sklearn/babyweight/trainer/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,5 +138,5 @@ def save_model(estimator, gcspath, name):
joblib.dump(estimator, model)
model_path = os.path.join(gcspath, datetime.datetime.now().strftime(
'export_%Y%m%d_%H%M%S'), model)
subprocess.check_call(['gsutil', 'cp', model, model_path])
subprocess.check_call(['gcloud', 'storage', 'cp', model, model_path])
return model_path
13 changes: 4 additions & 9 deletions blogs/sklearn/babyweight_skl.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,7 @@
"outputs": [],
"source": [
"%%bash\n",
"if ! gsutil ls | grep -q gs://${BUCKET}/; then\n",
" gsutil mb -l ${REGION} gs://${BUCKET}\n",
"fi"
"if ! gcloud storage ls | grep -q gs://${BUCKET}/; then\n", " gcloud storage buckets create --location ${REGION} gs://${BUCKET}\n", "fi"
]
},
{
Expand Down Expand Up @@ -840,8 +838,7 @@
" joblib.dump(estimator, model)\n",
" model_path = os.path.join(gcspath, datetime.datetime.now().strftime(\n",
" 'export_%Y%m%d_%H%M%S'), model)\n",
" subprocess.check_call(['gsutil', 'cp', model, model_path])\n",
" return model_path"
" subprocess.check_call(['gcloud', 'storage', 'cp', model, model_path])\n", " return model_path"
]
},
{
Expand Down Expand Up @@ -1199,8 +1196,7 @@
],
"source": [
"%bash\n",
"gsutil ls gs://${BUCKET}/babyweight/sklearn/ | tail -1"
]
"gcloud storage ls gs://${BUCKET}/babyweight/sklearn/ | tail -1" ]
},
{
"cell_type": "code",
Expand All @@ -1218,8 +1214,7 @@
"%bash\n",
"MODEL_NAME=\"babyweight\"\n",
"MODEL_VERSION=\"skl\"\n",
"MODEL_LOCATION=$(gsutil ls gs://${BUCKET}/babyweight/sklearn/ | tail -1)\n",
"echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
"MODEL_LOCATION=$(gcloud storage ls gs://${BUCKET}/babyweight/sklearn/ | tail -1)\n", "echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
"#gcloud ml-engine versions delete ${MODEL_VERSION} --model ${MODEL_NAME}\n",
"#gcloud ml-engine models delete ${MODEL_NAME}\n",
"#gcloud ml-engine models create ${MODEL_NAME} --regions $REGION\n",
Expand Down