diff --git a/Workspace_tutorial.ipynb b/Workspace_tutorial.ipynb index 57cd3a9..839dcee 100644 --- a/Workspace_tutorial.ipynb +++ b/Workspace_tutorial.ipynb @@ -1,623 +1,608 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "collapsed_sections": [ - "o3NsLMqUQnOF" - ], - "authorship_tag": "ABX9TyOTajJcyYgAnYRXXXMq+sml", - "include_colab_link": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "view-in-github", - "colab_type": "text" - }, - "source": [ - "\"Open" - ] - }, - { - "cell_type": "markdown", - "source": [ - "\"Weights\n", - "\n", - "\n", - "# Tutorial for [Programmatic Workspace](https://docs.wandb.ai/guides/app/pages/workspaces#saved-workspace-views)\n", - "\n", - "\n", - "### Overview\n", - "* [wandb-workspaces](https://github.com/wandb/wandb-workspaces/tree/main) is a Python library for programmatically working with Weights & Biases workspaces and Reports.\n", - "\n", - "* Programmatically create and customize workspaces by defining configurations, setting panel layouts, and organizing sections. Load and modify workspaces via URL, filter and group runs using expressions, and customize run appearances.\n", - "\n", - "### Known limitations\n", - "* Only compatible with [Saved Views](https://docs.wandb.ai/guides/app/pages/workspaces#workspace-types) (URLs ending with ?nw=...). Clone a `Personal Workspace` as a `Saved View` for programmatic interaction.\n", - "\n", - "### Notebook Instructions\n", - "* This notebook will create the project, log sample a to it, and then demonstrate the workspace examples using this data.\n", - "* Run each cell step by step. The notebook will log sample data to a project named `workspace-api-example`.\n" - ], - "metadata": { - "id": "MmxL0wjvrNtQ" - } - }, - { - "cell_type": "markdown", - "source": [ - "# 1. Install and Import Dependencies" - ], - "metadata": { - "id": "X6AencUCi19a" - } - }, - { - "cell_type": "code", - "source": [ - "# Install dependencies\n", - "!pip install wandb wandb-workspaces" - ], - "metadata": { - "id": "99KU0ZiY4t36" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "# Import dependencies\n", - "\n", - "import os\n", - "import wandb\n", - "import wandb_workspaces.workspaces as ws\n", - "import wandb_workspaces.reports.v2 as wr # We use the Reports API for adding panels\n", - "\n", - "# Improve output formatting\n", - "%load_ext rich" - ], - "metadata": { - "id": "IQigRhck4zbF" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# 2. Create a Project to interact with programmatically for notebook examples\n", - "\n", - "Create a new project or load an existing workspace with its unique `Saved view` URL. For this notebook, we will create a new workspace to use." - ], - "metadata": { - "id": "1CAls6JljDFo" - } - }, - { - "cell_type": "code", - "source": [ - "# Alternatively, you can load an existing view from a W&B via URL with ws.Workspace.from_url\n", - "\n", - "# wandb.login()\n", - "\n", - "# workspace = ws.Workspace.from_url(\"https://wandb.ai/[SOURCE-ENTITY]/[SOURCE-USER]?nw=abc\").\n", - "\n", - "# workspace = ws.Workspace(\n", - "# entity=\"NEW-ENTITY\",\n", - "# project=NEW-PROJECT,\n", - "# name=\"NEW-SAVED-VIEW-NAME\",)" - ], - "metadata": { - "id": "uZF7Zgx3uOfu" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "# Initialize Weights & Biases and Login\n", - "wandb.login()\n", - "\n", - "# Function to create a new project and log sample data\n", - "def create_project_and_log_data():\n", - " project = \"workspace-api-example\" # Default project name\n", - "\n", - " # Initialize a run to log some sample data\n", - " with wandb.init(project=project, name=\"sample_run\") as run:\n", - " for step in range(100):\n", - " wandb.log({\n", - " \"Step\": step,\n", - " \"val_loss\": 1.0 / (step + 1),\n", - " \"val_accuracy\": step / 100.0,\n", - " \"train_loss\": 1.0 / (step + 2),\n", - " \"train_accuracy\": step / 110.0,\n", - " \"f1_score\": step / 100.0,\n", - " \"recall\": step / 120.0,\n", - " })\n", - " return project\n", - "\n", - "# Create a new project and log data\n", - "project = create_project_and_log_data()\n", - "entity = wandb.Api().default_entity" - ], - "metadata": { - "id": "rApkZmZB9Ueg", - "collapsed": true - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# 3. Programmatic Workspace Examples\n", - "The API allows you to create, manage, and customize workspaces programmatically. These capabilities are designed to help you organize and visualize your machine learning experiments more effectively.\n", - "\n", - "Below are examples for using programmatic workspace features:" - ], - "metadata": { - "id": "E7WAoOlsiQTL" - } - }, - { - "cell_type": "code", - "source": [ - "# See all available settings for workspaces, sections, and panels.\n", - "all_settings_objects = [x for x in dir(ws) if isinstance(getattr(ws, x), type)]\n", - "all_settings_objects" - ], - "metadata": { - "id": "irVaxJ289yMa" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Creating a workspace `saved view`\n", - "\n", - "This example demonstrates how to create a new workspace and populate it with sections and panels. Workspaces can be edited like regular Python objects, providing flexibility and ease of use.\n", - "\n" - ], - "metadata": { - "id": "kHD8tEjEPqjL" - } - }, - { - "cell_type": "code", - "source": [ - "def sample_workspace_saved_example(entity: str, project: str) -> str:\n", - " workspace: ws.Workspace = ws.Workspace(\n", - " name=\"Example W&B Workspace\",\n", - " entity=entity,\n", - " project=project,\n", - " sections=[\n", - " ws.Section(\n", - " name=\"Validation Metrics\",\n", - " panels=[\n", - " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", - " wr.BarPlot(metrics=[\"val_accuracy\"]),\n", - " wr.ScalarChart(metric=\"f1_score\", groupby_aggfunc=\"mean\"),\n", - " ],\n", - " is_open=True,\n", - " ),\n", - " ],\n", - " )\n", - " workspace.save()\n", - " print(\"Sample Workspace saved.\")\n", - " return workspace.url\n", - "\n", - "workspace_url: str = sample_workspace_saved_example(entity, project)" - ], - "metadata": { - "id": "sWmN5lNJB-pH" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Loading a workspace from a URL\n", - "\n", - "This example demonstrates how to load an existing workspace and save it as a new view. This allows you to easily duplicate and customize workspaces without affecting the original setup." - ], - "metadata": { - "id": "SIIcxNVSQXmu" - } - }, - { - "cell_type": "code", - "source": [ - "def save_new_workspace_view_example(url: str) -> None:\n", - " workspace: ws.Workspace = ws.Workspace.from_url(url)\n", - "\n", - " workspace.name = \"Updated Workspace Name\"\n", - " workspace.save()\n", - "\n", - " print(f\"Workspace saved as new view.\")\n", - "\n", - "save_new_workspace_view_example(workspace_url)" - ], - "metadata": { - "id": "_gWiaq_r9xIM" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Basic Settings\n", - "\n", - "This example demonstrates how to create a workspace, add sections with panels, and configure settings for the workspace, individual sections, and panels.\n" - ], - "metadata": { - "id": "o3NsLMqUQnOF" - } - }, - { - "cell_type": "code", - "source": [ - "\n", - "# Function to create and configure a workspace with custom settings\n", - "def custom_settings_example(entity: str, project: str) -> None:\n", - " workspace: ws.Workspace = ws.Workspace(name=\"An example workspace\", entity=entity, project=project)\n", - " workspace.sections = [\n", - " ws.Section(\n", - " name=\"Validation\",\n", - " panels=[\n", - " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", - " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", - " wr.ScalarChart(metric=\"f1_score\", groupby_aggfunc=\"mean\"),\n", - " wr.ScalarChart(metric=\"recall\", groupby_aggfunc=\"mean\"),\n", - " ],\n", - " is_open=True,\n", - " ),\n", - " ws.Section(\n", - " name=\"Training\",\n", - " panels=[\n", - " wr.LinePlot(x=\"Step\", y=[\"train_loss\"]),\n", - " wr.LinePlot(x=\"Step\", y=[\"train_accuracy\"]),\n", - " ],\n", - " is_open=False,\n", - " ),\n", - " ]\n", - "\n", - " workspace.settings = ws.WorkspaceSettings(\n", - " x_axis=\"Step\",\n", - " x_min=0,\n", - " x_max=75,\n", - " smoothing_type=\"gaussian\",\n", - " smoothing_weight=20.0,\n", - " ignore_outliers=False,\n", - " remove_legends_from_panels=False,\n", - " tooltip_number_of_runs=\"default\",\n", - " tooltip_color_run_names=True,\n", - " max_runs=20,\n", - " point_visualization_method=\"bucketing\",\n", - " auto_expand_panel_search_results=False,\n", - " )\n", - "\n", - " section = workspace.sections[0]\n", - " section.panel_settings = ws.SectionPanelSettings(\n", - " x_min=25,\n", - " x_max=50,\n", - " smoothing_type=\"none\",\n", - " )\n", - "\n", - " panel = section.panels[0]\n", - " panel.title = \"Validation Loss Custom Title\"\n", - " panel.title_x = \"Custom x-axis title\"\n", - "\n", - " workspace.save()\n", - " print(\"Workspace with custom settings saved.\")\n", - "\n", - "# Run the function to create and configure the workspace\n", - "custom_settings_example(entity, project)" - ], - "metadata": { - "id": "Jeo-2lqMSKR5" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Filtering Runs\n", - "\n", - "This example demonstrates how to use run filters to templatize powerful filtering capabilities aligned with specific data patterns.\n", - "\n", - "\n" - ], - "metadata": { - "id": "oT54K4DLQos-" - } - }, - { - "cell_type": "code", - "source": [ - "def advanced_filter_example(entity: str, project: str) -> None:\n", - " # Get all runs in the project\n", - " runs: list = wandb.Api().runs(f\"{entity}/{project}\")\n", - "\n", - " # Apply multiple filters: val_loss < 0.1, val_accuracy > 0.8, and run name matches regex pattern\n", - " workspace: ws.Workspace = ws.Workspace(\n", - " name=\"Advanced Filtered Workspace with Regex\",\n", - " entity=entity,\n", - " project=project,\n", - " sections=[\n", - " ws.Section(\n", - " name=\"Advanced Filtered Section\",\n", - " panels=[\n", - " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", - " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", - " ],\n", - " is_open=True,\n", - " ),\n", - " ],\n", - " runset_settings=ws.RunsetSettings(\n", - " filters=[\n", - " (ws.Summary(\"val_loss\") < 0.1), # Filter runs by the 'val_loss' summary\n", - " (ws.Summary(\"val_accuracy\") > 0.8), # Filter runs by the 'val_accuracy' summary\n", - " (ws.Metric(\"ID\").isin([run.id for run in wandb.Api().runs(f\"{entity}/{project}\")])),\n", - " ],\n", - " regex_query=True,\n", - " )\n", - " )\n", - "\n", - " # Add regex search to match run names starting with 's'\n", - " workspace.runset_settings.query = \"^s\"\n", - " workspace.runset_settings.regex_query = True\n", - "\n", - " workspace.save()\n", - " print(\"Workspace with advanced filters and regex search saved.\")\n", - "\n", - "advanced_filter_example(entity, project)" - ], - "metadata": { - "id": "IaBvnHSISobu" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Changing Runs Colors\n", - "This example demonstrates how to change the colors of the runs in a workspace and allows users to programmatically distinguish between different runs by assigning unique colors." - ], - "metadata": { - "id": "-boRSciuQo1a" - } - }, - { - "cell_type": "code", - "source": [ - "def run_color_example(entity: str, project: str) -> None:\n", - " # Get all runs in the project\n", - " runs: list = wandb.Api().runs(f\"{entity}/{project}\")\n", - "\n", - " # Dynamically assign colors to the runs\n", - " run_colors: list = ['purple', 'orange', 'teal', 'magenta']\n", - " run_settings: dict = {}\n", - " for i, run in enumerate(runs):\n", - " run_settings[run.id] = ws.RunSettings(color=run_colors[i % len(run_colors)])\n", - "\n", - " workspace: ws.Workspace = ws.Workspace(\n", - " name=\"Run Colors Workspace\",\n", - " entity=entity,\n", - " project=project,\n", - " sections=[\n", - " ws.Section(\n", - " name=\"Run Colors Section\",\n", - " panels=[\n", - " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", - " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", - " ],\n", - " is_open=True,\n", - " ),\n", - " ],\n", - " runset_settings=ws.RunsetSettings(\n", - " run_settings=run_settings\n", - " )\n", - " )\n", - "\n", - " workspace.save()\n", - " print(\"Workspace with run colors saved.\")\n", - "\n", - "run_color_example(entity, project)" - ], - "metadata": { - "id": "6JnC9tFKbYgm" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Grouping Runs\n", - "\n", - "This example demonstrates how to group runs by specific metrics.\n" - ], - "metadata": { - "id": "09GiaVZCQo9Y" - } - }, - { - "cell_type": "code", - "source": [ - "def grouping_example(entity: str, project: str) -> None:\n", - " workspace: ws.Workspace = ws.Workspace(\n", - " name=\"Grouped Runs Workspace\",\n", - " entity=entity,\n", - " project=project,\n", - " sections=[\n", - " ws.Section(\n", - " name=\"Grouped Runs\",\n", - " panels=[\n", - " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", - " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", - " ],\n", - " is_open=True,\n", - " ),\n", - " ],\n", - " runset_settings=ws.RunsetSettings(\n", - " groupby=[ws.Metric(\"Name\")]\n", - " )\n", - " )\n", - " workspace.save()\n", - " print(\"Workspace with grouped runs saved.\")\n", - "\n", - "grouping_example(entity, project)" - ], - "metadata": { - "id": "oW4ZSYw5c1L_" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Sorting Runs\n", - "This example demonstrates how to sort runs by specific metrics." - ], - "metadata": { - "id": "3-soac7eOXRY" - } - }, - { - "cell_type": "code", - "source": [ - "def sorting_example(entity: str, project: str) -> None:\n", - " workspace: ws.Workspace = ws.Workspace(\n", - " name=\"Sorted Runs Workspace\",\n", - " entity=entity,\n", - " project=project,\n", - " sections=[\n", - " ws.Section(\n", - " name=\"Sorted Runs\",\n", - " panels=[\n", - " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", - " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", - " ],\n", - " is_open=True,\n", - " ),\n", - " ],\n", - " runset_settings=ws.RunsetSettings(\n", - " order=[ws.Ordering(ws.Summary(\"val_loss\"))] #Order using val_loss summary\n", - " )\n", - " )\n", - " workspace.save()\n", - " print(\"Workspace with sorted runs saved.\")\n", - "\n", - "sorting_example(entity, project)" - ], - "metadata": { - "id": "UbIqEPtJdFR5" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# 4. Putting it all together: comprehenive example\n", - "\n", - "This example demonstrates how to create a comprehensive workspace, configure its settings, and add panels to sections." - ], - "metadata": { - "id": "3BwWS8b_QpMI" - } - }, - { - "cell_type": "code", - "source": [ - "def full_end_to_end_example(entity: str, project: str) -> None:\n", - " # Get all runs in the project\n", - " runs: list = wandb.Api().runs(f\"{entity}/{project}\")\n", - "\n", - " # Dynamically assign colors to the runs and create run settings\n", - " run_colors: list = ['red', 'blue', 'green', 'orange', 'purple', 'teal', 'magenta', '#FAC13C']\n", - " run_settings: dict = {}\n", - " for i, run in enumerate(runs):\n", - " run_settings[run.id] = ws.RunSettings(color=run_colors[i % len(run_colors)], disabled=False)\n", - "\n", - " workspace: ws.Workspace = ws.Workspace(\n", - " name=\"My Workspace Template\",\n", - " entity=entity,\n", - " project=project,\n", - " sections=[\n", - " ws.Section(\n", - " name=\"Main Metrics\",\n", - " panels=[\n", - " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", - " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", - " wr.ScalarChart(metric=\"f1_score\", groupby_aggfunc=\"mean\"),\n", - " ],\n", - " is_open=True,\n", - " ),\n", - " ws.Section(\n", - " name=\"Additional Metrics\",\n", - " panels=[\n", - " wr.ScalarChart(metric=\"precision\", groupby_aggfunc=\"mean\"),\n", - " wr.ScalarChart(metric=\"recall\", groupby_aggfunc=\"mean\"),\n", - " ],\n", - " ),\n", - " ],\n", - " settings=ws.WorkspaceSettings(\n", - " x_axis=\"Step\",\n", - " x_min=0,\n", - " x_max=100,\n", - " smoothing_type=\"none\",\n", - " smoothing_weight=0,\n", - " ignore_outliers=False,\n", - " remove_legends_from_panels=False,\n", - " tooltip_number_of_runs=\"default\",\n", - " tooltip_color_run_names=True,\n", - " max_runs=20,\n", - " point_visualization_method=\"bucketing\",\n", - " auto_expand_panel_search_results=False,\n", - " ),\n", - " runset_settings=ws.RunsetSettings(\n", - " query=\"\",\n", - " regex_query=False,\n", - " filters=[\n", - " ws.Summary(\"val_loss\") < 1,\n", - " ws.Metric(\"Name\") == \"sample_run\",\n", - " ],\n", - " groupby=[ws.Metric(\"Name\")],\n", - " order=[ws.Ordering(ws.Summary(\"Step\"), ascending=True)],\n", - " run_settings=run_settings\n", - " )\n", - " )\n", - " workspace.save()\n", - " print(\"Workspace created and saved.\")\n", - "\n", - "full_end_to_end_example(entity, project)" - ], - "metadata": { - "id": "aztgJmIPdVBl" - }, - "execution_count": null, - "outputs": [] - } - ] -} \ No newline at end of file + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Weights\n", + "\n", + "\n", + "# Programmatic Workspaces\n", + "\n", + "Organize and visualize your machine learning experiments more effectively by programmatically create, manage, and customize workspaces by defining configurations, setting panel layouts, and organizing sections with the [`wandb-workspaces`](https://github.com/wandb/wandb-workspaces/tree/main) W&B library. Load and modify workspaces with URLs, filter and group runs using expressions, and customize run appearances.\n", + "\n", + "`wandb-workspaces` is a Python library for programmatically creating and customizing W&B [Workspaces](https://docs.wandb.ai/guides/app/pages/workspaces) and [Reports](https://docs.wandb.ai/guides/reports).\n", + "\n", + "In this tutorial you will see how to use `wandb-workspaces` to create and customize workspaces by defining configurations, set panel layouts, and organize sections.\n", + "\n", + "\n", + "### How to use this notebook\n", + "* Run each cell one at a time. \n", + "* Copy and paste the URL that is printed after you run a cell to view the changes made to the workspace.\n", + "\n", + "\n", + "### Note\n", + "Programmatic interaction with workspaces is currently supported for [**Saved workspaces views**](https://docs.wandb.ai/guides/app/pages/workspaces#saved-workspace-views). Saved workspaces views are collaborative snapshots of a workspace. Anyone on your team can view, edit, and save changes to saved workspace views. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Install and import dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install dependencies\n", + "!pip install wandb wandb-workspaces rich" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import dependencies\n", + "import os\n", + "import wandb\n", + "import wandb_workspaces.workspaces as ws\n", + "import wandb_workspaces.reports.v2 as wr # We use the Reports API for adding panels\n", + "\n", + "# Improve output formatting\n", + "%load_ext rich" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Create a new project and workspace\n", + "\n", + "For this tutorial we will create a new project so that we can experiment with the `wandb_workspaces` API:\n", + "\n", + "Note: You can load an existing workspace using its unique `Saved view` URL. See the next code block to see how to do this." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Weights & Biases and Login\n", + "wandb.login()\n", + "\n", + "# Function to create a new project and log sample data\n", + "def create_project_and_log_data():\n", + " project = \"workspace-api-example\" # Default project name\n", + "\n", + " # Initialize a run to log some sample data\n", + " with wandb.init(project=project, name=\"sample_run\") as run:\n", + " for step in range(100):\n", + " wandb.log({\n", + " \"Step\": step,\n", + " \"val_loss\": 1.0 / (step + 1),\n", + " \"val_accuracy\": step / 100.0,\n", + " \"train_loss\": 1.0 / (step + 2),\n", + " \"train_accuracy\": step / 110.0,\n", + " \"f1_score\": step / 100.0,\n", + " \"recall\": step / 120.0,\n", + " })\n", + " return project\n", + "\n", + "# Create a new project and log data\n", + "project = create_project_and_log_data()\n", + "entity = wandb.Api().default_entity" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### (Optional) Load an existing project and workspace\n", + "Instead of creating a new project, you can load one of your own existing project and workspace. To do this, find the unique workspace URL and pass it to `ws.Workspace.from_url` as a string. The URL has the form `https://wandb.ai/[SOURCE-ENTITY]/[SOURCE-USER]?nw=abc`. \n", + "\n", + "For example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# wandb.login()\n", + "\n", + "# workspace = ws.Workspace.from_url(\"https://wandb.ai/[SOURCE-ENTITY]/[SOURCE-USER]?nw=abc\").\n", + "\n", + "# workspace = ws.Workspace(\n", + "# entity=\"NEW-ENTITY\",\n", + "# project=NEW-PROJECT,\n", + "# name=\"NEW-SAVED-VIEW-NAME\"\n", + "# )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Programmatic workspace examples\n", + "Below are examples for using programmatic workspace features:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# See all available settings for workspaces, sections, and panels.\n", + "all_settings_objects = [x for x in dir(ws) if isinstance(getattr(ws, x), type)]\n", + "all_settings_objects" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a workspace with `saved view`\n", + "This example demonstrates how to create a new workspace and populate it with sections and panels. Workspaces can be edited like regular Python objects, providing flexibility and ease of use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def sample_workspace_saved_example(entity: str, project: str) -> str:\n", + " workspace: ws.Workspace = ws.Workspace(\n", + " name=\"Example W&B Workspace\",\n", + " entity=entity,\n", + " project=project,\n", + " sections=[\n", + " ws.Section(\n", + " name=\"Validation Metrics\",\n", + " panels=[\n", + " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", + " wr.BarPlot(metrics=[\"val_accuracy\"]),\n", + " wr.ScalarChart(metric=\"f1_score\", groupby_aggfunc=\"mean\"),\n", + " ],\n", + " is_open=True,\n", + " ),\n", + " ],\n", + " )\n", + " workspace.save()\n", + " print(\"Sample Workspace saved.\")\n", + " return workspace.url\n", + "\n", + "workspace_url: str = sample_workspace_saved_example(entity, project)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load a workspace from a URL\n", + "Duplicate and customize workspaces without affecting the original setup. To do this, load an existing workspace and save it as a new view:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def save_new_workspace_view_example(url: str) -> None:\n", + " workspace: ws.Workspace = ws.Workspace.from_url(url)\n", + "\n", + " workspace.name = \"Updated Workspace Name\"\n", + " workspace.save()\n", + "\n", + " print(f\"Workspace saved as new view.\")\n", + "\n", + "save_new_workspace_view_example(workspace_url)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that your workspace is now named \"Updated Workspace Name\"." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Basic settings\n", + "The following code shows how to create a workspace, add sections with panels, and configure settings for the workspace, individual sections, and panels:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Function to create and configure a workspace with custom settings\n", + "def custom_settings_example(entity: str, project: str) -> None:\n", + " workspace: ws.Workspace = ws.Workspace(name=\"An example workspace\", entity=entity, project=project)\n", + " workspace.sections = [\n", + " ws.Section(\n", + " name=\"Validation\",\n", + " panels=[\n", + " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", + " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", + " wr.ScalarChart(metric=\"f1_score\", groupby_aggfunc=\"mean\"),\n", + " wr.ScalarChart(metric=\"recall\", groupby_aggfunc=\"mean\"),\n", + " ],\n", + " is_open=True,\n", + " ),\n", + " ws.Section(\n", + " name=\"Training\",\n", + " panels=[\n", + " wr.LinePlot(x=\"Step\", y=[\"train_loss\"]),\n", + " wr.LinePlot(x=\"Step\", y=[\"train_accuracy\"]),\n", + " ],\n", + " is_open=False,\n", + " ),\n", + " ]\n", + "\n", + " workspace.settings = ws.WorkspaceSettings(\n", + " x_axis=\"Step\",\n", + " x_min=0,\n", + " x_max=75,\n", + " smoothing_type=\"gaussian\",\n", + " smoothing_weight=20.0,\n", + " ignore_outliers=False,\n", + " remove_legends_from_panels=False,\n", + " tooltip_number_of_runs=\"default\",\n", + " tooltip_color_run_names=True,\n", + " max_runs=20,\n", + " point_visualization_method=\"bucketing\",\n", + " auto_expand_panel_search_results=False,\n", + " )\n", + "\n", + " section = workspace.sections[0]\n", + " section.panel_settings = ws.SectionPanelSettings(\n", + " x_min=25,\n", + " x_max=50,\n", + " smoothing_type=\"none\",\n", + " )\n", + "\n", + " panel = section.panels[0]\n", + " panel.title = \"Validation Loss Custom Title\"\n", + " panel.title_x = \"Custom x-axis title\"\n", + "\n", + " workspace.save()\n", + " print(\"Workspace with custom settings saved.\")\n", + "\n", + "# Run the function to create and configure the workspace\n", + "custom_settings_example(entity, project)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that you are now viewing a different saved view called \"An example workspace\"." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Customize runs\n", + "The following code cells show you how to filter, change the color, group, and sort runs programmatically. \n", + "\n", + "In each example, the general workflow is to specify the desired customization as an argument to the appropiate parameter in `ws.RunsetSettings`.\n", + "\n", + "### Filter runs\n", + "You can create filters with python expressions and metrics you log with `wandb.log` or that are logged automatically as part of the run such as **Created Timestamp**. You can also reference filters by how they appear in the W&B App UI such as the **Name**, **Tags**, or **ID**.\n", + "\n", + "The following example shows how to filter runs based on the validation loss summary, validation accuracy summary, and the regex specified:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def advanced_filter_example(entity: str, project: str) -> None:\n", + " # Get all runs in the project\n", + " runs: list = wandb.Api().runs(f\"{entity}/{project}\")\n", + "\n", + " # Apply multiple filters: val_loss < 0.1, val_accuracy > 0.8, and run name matches regex pattern\n", + " workspace: ws.Workspace = ws.Workspace(\n", + " name=\"Advanced Filtered Workspace with Regex\",\n", + " entity=entity,\n", + " project=project,\n", + " sections=[\n", + " ws.Section(\n", + " name=\"Advanced Filtered Section\",\n", + " panels=[\n", + " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", + " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", + " ],\n", + " is_open=True,\n", + " ),\n", + " ],\n", + " runset_settings=ws.RunsetSettings(\n", + " filters=[\n", + " (ws.Summary(\"val_loss\") < 0.1), # Filter runs by the 'val_loss' summary\n", + " (ws.Summary(\"val_accuracy\") > 0.8), # Filter runs by the 'val_accuracy' summary\n", + " (ws.Metric(\"ID\").isin([run.id for run in wandb.Api().runs(f\"{entity}/{project}\")])),\n", + " ],\n", + " regex_query=True,\n", + " )\n", + " )\n", + "\n", + " # Add regex search to match run names starting with 's'\n", + " workspace.runset_settings.query = \"^s\"\n", + " workspace.runset_settings.regex_query = True\n", + "\n", + " workspace.save()\n", + " print(\"Workspace with advanced filters and regex search saved.\")\n", + "\n", + "advanced_filter_example(entity, project)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that passing in a list of filter expressions applies the boolean \"AND\" logic." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Change the colors of runs\n", + "This example demonstrates how to change the colors of the runs in a workspace:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def run_color_example(entity: str, project: str) -> None:\n", + " # Get all runs in the project\n", + " runs: list = wandb.Api().runs(f\"{entity}/{project}\")\n", + "\n", + " # Dynamically assign colors to the runs\n", + " run_colors: list = ['purple', 'orange', 'teal', 'magenta']\n", + " run_settings: dict = {}\n", + " for i, run in enumerate(runs):\n", + " run_settings[run.id] = ws.RunSettings(color=run_colors[i % len(run_colors)])\n", + "\n", + " workspace: ws.Workspace = ws.Workspace(\n", + " name=\"Run Colors Workspace\",\n", + " entity=entity,\n", + " project=project,\n", + " sections=[\n", + " ws.Section(\n", + " name=\"Run Colors Section\",\n", + " panels=[\n", + " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", + " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", + " ],\n", + " is_open=True,\n", + " ),\n", + " ],\n", + " runset_settings=ws.RunsetSettings(\n", + " run_settings=run_settings\n", + " )\n", + " )\n", + "\n", + " workspace.save()\n", + " print(\"Workspace with run colors saved.\")\n", + "\n", + "run_color_example(entity, project)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Group runs\n", + "\n", + "This example demonstrates how to group runs by specific metrics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def grouping_example(entity: str, project: str) -> None:\n", + " workspace: ws.Workspace = ws.Workspace(\n", + " name=\"Grouped Runs Workspace\",\n", + " entity=entity,\n", + " project=project,\n", + " sections=[\n", + " ws.Section(\n", + " name=\"Grouped Runs\",\n", + " panels=[\n", + " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", + " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", + " ],\n", + " is_open=True,\n", + " ),\n", + " ],\n", + " runset_settings=ws.RunsetSettings(\n", + " groupby=[ws.Metric(\"Name\")]\n", + " )\n", + " )\n", + " workspace.save()\n", + " print(\"Workspace with grouped runs saved.\")\n", + "\n", + "grouping_example(entity, project)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Sort runs\n", + "This example demonstrates how to sort runs based on the validation loss summary:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def sorting_example(entity: str, project: str) -> None:\n", + " workspace: ws.Workspace = ws.Workspace(\n", + " name=\"Sorted Runs Workspace\",\n", + " entity=entity,\n", + " project=project,\n", + " sections=[\n", + " ws.Section(\n", + " name=\"Sorted Runs\",\n", + " panels=[\n", + " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", + " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", + " ],\n", + " is_open=True,\n", + " ),\n", + " ],\n", + " runset_settings=ws.RunsetSettings(\n", + " order=[ws.Ordering(ws.Summary(\"val_loss\"))] #Order using val_loss summary\n", + " )\n", + " )\n", + " workspace.save()\n", + " print(\"Workspace with sorted runs saved.\")\n", + "\n", + "sorting_example(entity, project)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Putting it all together: comprehenive example\n", + "\n", + "This example demonstrates how to create a comprehensive workspace, configure its settings, and add panels to sections:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def full_end_to_end_example(entity: str, project: str) -> None:\n", + " # Get all runs in the project\n", + " runs: list = wandb.Api().runs(f\"{entity}/{project}\")\n", + "\n", + " # Dynamically assign colors to the runs and create run settings\n", + " run_colors: list = ['red', 'blue', 'green', 'orange', 'purple', 'teal', 'magenta', '#FAC13C']\n", + " run_settings: dict = {}\n", + " for i, run in enumerate(runs):\n", + " run_settings[run.id] = ws.RunSettings(color=run_colors[i % len(run_colors)], disabled=False)\n", + "\n", + " workspace: ws.Workspace = ws.Workspace(\n", + " name=\"My Workspace Template\",\n", + " entity=entity,\n", + " project=project,\n", + " sections=[\n", + " ws.Section(\n", + " name=\"Main Metrics\",\n", + " panels=[\n", + " wr.LinePlot(x=\"Step\", y=[\"val_loss\"]),\n", + " wr.LinePlot(x=\"Step\", y=[\"val_accuracy\"]),\n", + " wr.ScalarChart(metric=\"f1_score\", groupby_aggfunc=\"mean\"),\n", + " ],\n", + " is_open=True,\n", + " ),\n", + " ws.Section(\n", + " name=\"Additional Metrics\",\n", + " panels=[\n", + " wr.ScalarChart(metric=\"precision\", groupby_aggfunc=\"mean\"),\n", + " wr.ScalarChart(metric=\"recall\", groupby_aggfunc=\"mean\"),\n", + " ],\n", + " ),\n", + " ],\n", + " settings=ws.WorkspaceSettings(\n", + " x_axis=\"Step\",\n", + " x_min=0,\n", + " x_max=100,\n", + " smoothing_type=\"none\",\n", + " smoothing_weight=0,\n", + " ignore_outliers=False,\n", + " remove_legends_from_panels=False,\n", + " tooltip_number_of_runs=\"default\",\n", + " tooltip_color_run_names=True,\n", + " max_runs=20,\n", + " point_visualization_method=\"bucketing\",\n", + " auto_expand_panel_search_results=False,\n", + " ),\n", + " runset_settings=ws.RunsetSettings(\n", + " query=\"\",\n", + " regex_query=False,\n", + " filters=[\n", + " ws.Summary(\"val_loss\") < 1,\n", + " ws.Metric(\"Name\") == \"sample_run\",\n", + " ],\n", + " groupby=[ws.Metric(\"Name\")],\n", + " order=[ws.Ordering(ws.Summary(\"Step\"), ascending=True)],\n", + " run_settings=run_settings\n", + " )\n", + " )\n", + " workspace.save()\n", + " print(\"Workspace created and saved.\")\n", + "\n", + "full_end_to_end_example(entity, project)" + ] + } + ], + "metadata": { + "colab": { + "authorship_tag": "ABX9TyOTajJcyYgAnYRXXXMq+sml", + "collapsed_sections": [ + "o3NsLMqUQnOF" + ], + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}