Sync offline projects to Hugging Face spaces (#343)

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

.changeset/fair-lies-flow.md

@@ -0,0 +1,5 @@
+---
+"trackio": minor
+---
+
+feat:Sync offline projects to Hugging Face spaces

docs/source/quickstart.md

@@ -108,3 +108,24 @@ trackio.init(project="my-project", space_id="username/space_id")
 ```
 
 it will use an existing or automatically deploy a new Hugging Face Space as needed. You should be logged in with the `huggingface-cli` locally and your token should have write permissions to create the Space.
+
+Alternatively, you can sync an existing local project to a Hugging Face Space by running:
+
+<hfoptions id="language">
+<hfoption id="Shell">
+
+```sh
+trackio sync --project "my-project" --space-id "username/space_id"
+```
+
+</hfoption>
+<hfoption id="Python">
+
+```py
+trackio.sync(project="my-project", space_id="username/space_id")
+```
+
+</hfoption>
+</hfoptions>
+
+This will create the Space if it does not already exist, and upload all runs and associated data to the Space.

trackio/__init__.py

@@ -15,6 +15,7 @@
 from huggingface_hub import SpaceStorage
 
 from trackio import context_vars, deploy, utils
+from trackio.deploy import sync
 from trackio.histogram import Histogram
 from trackio.imports import import_csv, import_tf_events
 from trackio.media import TrackioAudio, TrackioImage, TrackioVideo
@@ -42,6 +43,7 @@
     "log",
     "finish",
     "show",
+    "sync",
     "delete_project",
     "import_csv",
     "import_tf_events",

trackio/cli.py

@@ -1,6 +1,6 @@
 import argparse
 
-from trackio import show
+from trackio import show, sync
 
 
 def main():
@@ -30,13 +30,43 @@ def main():
         help="Comma-separated list of hex color codes for plot lines (e.g. '#FF0000,#00FF00,#0000FF'). If not provided, the TRACKIO_COLOR_PALETTE environment variable will be used, or the default palette if not set.",
     )
 
+    sync_parser = subparsers.add_parser(
+        "sync",
+        help="Sync a local project's database to a Hugging Face Space. If the Space does not exist, it will be created.",
+    )
+    sync_parser.add_argument(
+        "--project", required=True, help="The name of the local project."
+    )
+    sync_parser.add_argument(
+        "--space-id",
+        required=True,
+        help="The Hugging Face Space ID where the project will be synced (e.g. username/space_id).",
+    )
+    sync_parser.add_argument(
+        "--private",
+        action="store_true",
+        help="Make the Hugging Face Space private if creating a new Space. By default, the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.",
+    )
+    sync_parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Overwrite the existing database without prompting for confirmation.",
+    )
+
     args = parser.parse_args()
 
     if args.command == "show":
         color_palette = None
         if args.color_palette:
             color_palette = [color.strip() for color in args.color_palette.split(",")]
         show(args.project, args.theme, args.mcp_server, color_palette)
+    elif args.command == "sync":
+        sync(
+            project=args.project,
+            space_id=args.space_id,
+            private=args.private,
+            force=args.force,
+        )
     else:
         parser.print_help()
 

trackio/deploy.py

@@ -14,6 +14,7 @@
 
 import trackio
 from trackio.sqlite_storage import SQLiteStorage
+from trackio.utils import preprocess_space_and_dataset_ids
 
 SPACE_HOST_URL = "https://{user_name}-{space_name}.hf.space/"
 SPACE_URL = "https://huggingface.co/spaces/{space_id}"
@@ -154,6 +155,8 @@ def deploy_as_space(
     if theme := os.environ.get("TRACKIO_THEME"):
         huggingface_hub.add_space_variable(space_id, "TRACKIO_THEME", theme)
 
+    huggingface_hub.add_space_variable(space_id, "GRADIO_MCP_SERVER", "True")
+
 
 def create_space_if_not_exists(
     space_id: str,
@@ -229,30 +232,75 @@ def wait_until_space_exists(
     Args:
         space_id: The ID of the Space to wait for.
     """
+    hf_api = huggingface_hub.HfApi()
     delay = 1
-    for _ in range(10):
+    for _ in range(30):
         try:
-            Client(space_id, verbose=False)
+            hf_api.space_info(space_id)
             return
-        except (ReadTimeout, ValueError):
+        except (huggingface_hub.utils.HfHubHTTPError, ReadTimeout):
             time.sleep(delay)
-            delay = min(delay * 2, 30)
+            delay = min(delay * 2, 60)
     raise TimeoutError("Waiting for space to exist took longer than expected")
 
 
-def upload_db_to_space(project: str, space_id: str) -> None:
+def upload_db_to_space(project: str, space_id: str, force: bool = False) -> None:
     """
-    Uploads the database of a local Trackio project to a Hugging Face Space.
+    Uploads the database of a local Trackio project to a Hugging Face Space. It
+    uses the Gradio Client to upload since we do not want to trigger a new build
+    of the Space, which would happen if we used `huggingface_hub.upload_file`.
 
     Args:
         project: The name of the project to upload.
         space_id: The ID of the Space to upload to.
+        force: If True, overwrite existing database without prompting. If False, prompt for confirmation.
     """
     db_path = SQLiteStorage.get_project_db_path(project)
-    client = Client(space_id, verbose=False)
+    client = Client(space_id, verbose=False, httpx_kwargs={"timeout": 90})
+
+    if not force:
+        try:
+            existing_projects = client.predict(api_name="/get_all_projects")
+            if project in existing_projects:
+                response = input(
+                    f"Database for project '{project}' already exists on Space '{space_id}'. "
+                    f"Overwrite it? (y/N): "
+                )
+                if response.lower() not in ["y", "yes"]:
+                    print("* Upload cancelled.")
+                    return
+        except Exception as e:
+            print(f"* Warning: Could not check if project exists on Space: {e}")
+            print("* Proceeding with upload...")
+
     client.predict(
         api_name="/upload_db_to_space",
         project=project,
         uploaded_db=handle_file(db_path),
         hf_token=huggingface_hub.utils.get_token(),
     )
+
+
+def sync(
+    project: str, space_id: str, private: bool | None = None, force: bool = False
+) -> None:
+    """
+    Syncs a local Trackio project's database to a Hugging Face Space.
+    If the Space does not exist, it will be created.
+
+    Args:
+        project (`str`): The name of the project to upload.
+        space_id (`str`): The ID of the Space to upload to (e.g., `"username/space_id"`).
+        private (`bool`, *optional*):
+            Whether to make the Space private. If None (default), the repo will be
+            public unless the organization's default is private. This value is ignored
+            if the repo already exists.
+        force (`bool`, *optional*, defaults to `False`):
+            If `True`, overwrite the existing database without prompting for confirmation.
+            If `False`, prompt the user before overwriting an existing database.
+    """
+    space_id, _ = preprocess_space_and_dataset_ids(space_id, None)
+    create_space_if_not_exists(space_id, private=private)
+    wait_until_space_exists(space_id)
+    upload_db_to_space(project, space_id, force=force)
+    print(f"Synced successfully to space: {SPACE_URL.format(space_id=space_id)}")

trackio/imports.py

@@ -14,6 +14,7 @@ def import_csv(
     space_id: str | None = None,
     dataset_id: str | None = None,
     private: bool | None = None,
+    force: bool = False,
 ) -> None:
     """
     Imports a CSV file into a Trackio project. The CSV file must contain a `"step"`
@@ -143,7 +144,7 @@ def import_csv(
             space_id=space_id, dataset_id=dataset_id, private=private
         )
         deploy.wait_until_space_exists(space_id=space_id)
-        deploy.upload_db_to_space(project=project, space_id=space_id)
+        deploy.upload_db_to_space(project=project, space_id=space_id, force=force)
         print(
             f"* View dashboard by going to: {deploy.SPACE_URL.format(space_id=space_id)}"
         )
@@ -156,6 +157,7 @@ def import_tf_events(
     space_id: str | None = None,
     dataset_id: str | None = None,
     private: bool | None = None,
+    force: bool = False,
 ) -> None:
     """
     Imports TensorFlow Events files from a directory into a Trackio project. Each
@@ -296,7 +298,7 @@ def import_tf_events(
             space_id, dataset_id=dataset_id, private=private
         )
         deploy.wait_until_space_exists(space_id)
-        deploy.upload_db_to_space(project, space_id)
+        deploy.upload_db_to_space(project, space_id, force=force)
         print(
             f"* View dashboard by going to: {deploy.SPACE_URL.format(space_id=space_id)}"
         )

trackio/ui/main.py

@@ -99,6 +99,18 @@ def get_runs(project) -> list[str]:
     return SQLiteStorage.get_runs(project)
 
 
+def upload_db_to_space(
+    project: str, uploaded_db: gr.FileData, hf_token: str | None
+) -> None:
+    """
+    Uploads the database of a local Trackio project to a Hugging Face Space.
+    """
+    fns.check_hf_token_has_write_access(hf_token)
+    db_project_path = SQLiteStorage.get_project_db_path(project)
+    os.makedirs(os.path.dirname(db_project_path), exist_ok=True)
+    shutil.copy(uploaded_db["path"], db_project_path)
+
+
 def get_available_metrics(project: str, runs: list[str]) -> list[str]:
     """Get all available metrics across all runs for x-axis selection."""
     if not project or not runs:
@@ -285,22 +297,6 @@ def toggle_timer(cb_value):
         return gr.Timer(active=False)
 
 
-def upload_db_to_space(
-    project: str, uploaded_db: gr.FileData, hf_token: str | None
-) -> None:
-    """
-    Uploads the database of a local Trackio project to a Hugging Face Space.
-    """
-    fns.check_hf_token_has_write_access(hf_token)
-    db_project_path = SQLiteStorage.get_project_db_path(project)
-    if os.path.exists(db_project_path):
-        raise gr.Error(
-            f"Trackio database file already exists for project {project}, cannot overwrite."
-        )
-    os.makedirs(os.path.dirname(db_project_path), exist_ok=True)
-    shutil.copy(uploaded_db["path"], db_project_path)
-
-
 def bulk_upload_media(uploads: list[UploadEntry], hf_token: str | None) -> None:
     """
     Uploads media files to a Trackio dashboard. Each entry in the list is a tuple of the project, run, and media file to be uploaded.