From 51bea30f2877adff8e6497466d3a799400a0a049 Mon Sep 17 00:00:00 2001 From: Can Demircan Date: Sat, 15 Nov 2025 00:00:10 +0100 Subject: [PATCH] Sync offline projects to Hugging Face spaces (#343) Co-authored-by: gradio-pr-bot Co-authored-by: Abubakar Abid --- .changeset/fair-lies-flow.md | 5 +++ docs/source/quickstart.md | 21 ++++++++++++ trackio/__init__.py | 2 ++ trackio/cli.py | 32 ++++++++++++++++++- trackio/deploy.py | 62 ++++++++++++++++++++++++++++++++---- trackio/imports.py | 6 ++-- trackio/ui/main.py | 28 +++++++--------- 7 files changed, 130 insertions(+), 26 deletions(-) create mode 100644 .changeset/fair-lies-flow.md diff --git a/.changeset/fair-lies-flow.md b/.changeset/fair-lies-flow.md new file mode 100644 index 0000000..10162aa --- /dev/null +++ b/.changeset/fair-lies-flow.md @@ -0,0 +1,5 @@ +--- +"trackio": minor +--- + +feat:Sync offline projects to Hugging Face spaces diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index 64b947d..4aff8f4 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -108,3 +108,24 @@ trackio.init(project="my-project", space_id="username/space_id") ``` it will use an existing or automatically deploy a new Hugging Face Space as needed. You should be logged in with the `huggingface-cli` locally and your token should have write permissions to create the Space. + +Alternatively, you can sync an existing local project to a Hugging Face Space by running: + + + + +```sh +trackio sync --project "my-project" --space-id "username/space_id" +``` + + + + +```py +trackio.sync(project="my-project", space_id="username/space_id") +``` + + + + +This will create the Space if it does not already exist, and upload all runs and associated data to the Space. \ No newline at end of file diff --git a/trackio/__init__.py b/trackio/__init__.py index 7e0905f..713fc47 100644 --- a/trackio/__init__.py +++ b/trackio/__init__.py @@ -15,6 +15,7 @@ from huggingface_hub import SpaceStorage from trackio import context_vars, deploy, utils +from trackio.deploy import sync from trackio.histogram import Histogram from trackio.imports import import_csv, import_tf_events from trackio.media import TrackioAudio, TrackioImage, TrackioVideo @@ -42,6 +43,7 @@ "log", "finish", "show", + "sync", "delete_project", "import_csv", "import_tf_events", diff --git a/trackio/cli.py b/trackio/cli.py index c53ae0c..cae24b3 100644 --- a/trackio/cli.py +++ b/trackio/cli.py @@ -1,6 +1,6 @@ import argparse -from trackio import show +from trackio import show, sync def main(): @@ -30,6 +30,29 @@ def main(): help="Comma-separated list of hex color codes for plot lines (e.g. '#FF0000,#00FF00,#0000FF'). If not provided, the TRACKIO_COLOR_PALETTE environment variable will be used, or the default palette if not set.", ) + sync_parser = subparsers.add_parser( + "sync", + help="Sync a local project's database to a Hugging Face Space. If the Space does not exist, it will be created.", + ) + sync_parser.add_argument( + "--project", required=True, help="The name of the local project." + ) + sync_parser.add_argument( + "--space-id", + required=True, + help="The Hugging Face Space ID where the project will be synced (e.g. username/space_id).", + ) + sync_parser.add_argument( + "--private", + action="store_true", + help="Make the Hugging Face Space private if creating a new Space. By default, the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.", + ) + sync_parser.add_argument( + "--force", + action="store_true", + help="Overwrite the existing database without prompting for confirmation.", + ) + args = parser.parse_args() if args.command == "show": @@ -37,6 +60,13 @@ def main(): if args.color_palette: color_palette = [color.strip() for color in args.color_palette.split(",")] show(args.project, args.theme, args.mcp_server, color_palette) + elif args.command == "sync": + sync( + project=args.project, + space_id=args.space_id, + private=args.private, + force=args.force, + ) else: parser.print_help() diff --git a/trackio/deploy.py b/trackio/deploy.py index 29d7b6e..be8e776 100644 --- a/trackio/deploy.py +++ b/trackio/deploy.py @@ -14,6 +14,7 @@ import trackio from trackio.sqlite_storage import SQLiteStorage +from trackio.utils import preprocess_space_and_dataset_ids SPACE_HOST_URL = "https://{user_name}-{space_name}.hf.space/" SPACE_URL = "https://huggingface.co/spaces/{space_id}" @@ -154,6 +155,8 @@ def deploy_as_space( if theme := os.environ.get("TRACKIO_THEME"): huggingface_hub.add_space_variable(space_id, "TRACKIO_THEME", theme) + huggingface_hub.add_space_variable(space_id, "GRADIO_MCP_SERVER", "True") + def create_space_if_not_exists( space_id: str, @@ -229,30 +232,75 @@ def wait_until_space_exists( Args: space_id: The ID of the Space to wait for. """ + hf_api = huggingface_hub.HfApi() delay = 1 - for _ in range(10): + for _ in range(30): try: - Client(space_id, verbose=False) + hf_api.space_info(space_id) return - except (ReadTimeout, ValueError): + except (huggingface_hub.utils.HfHubHTTPError, ReadTimeout): time.sleep(delay) - delay = min(delay * 2, 30) + delay = min(delay * 2, 60) raise TimeoutError("Waiting for space to exist took longer than expected") -def upload_db_to_space(project: str, space_id: str) -> None: +def upload_db_to_space(project: str, space_id: str, force: bool = False) -> None: """ - Uploads the database of a local Trackio project to a Hugging Face Space. + Uploads the database of a local Trackio project to a Hugging Face Space. It + uses the Gradio Client to upload since we do not want to trigger a new build + of the Space, which would happen if we used `huggingface_hub.upload_file`. Args: project: The name of the project to upload. space_id: The ID of the Space to upload to. + force: If True, overwrite existing database without prompting. If False, prompt for confirmation. """ db_path = SQLiteStorage.get_project_db_path(project) - client = Client(space_id, verbose=False) + client = Client(space_id, verbose=False, httpx_kwargs={"timeout": 90}) + + if not force: + try: + existing_projects = client.predict(api_name="/get_all_projects") + if project in existing_projects: + response = input( + f"Database for project '{project}' already exists on Space '{space_id}'. " + f"Overwrite it? (y/N): " + ) + if response.lower() not in ["y", "yes"]: + print("* Upload cancelled.") + return + except Exception as e: + print(f"* Warning: Could not check if project exists on Space: {e}") + print("* Proceeding with upload...") + client.predict( api_name="/upload_db_to_space", project=project, uploaded_db=handle_file(db_path), hf_token=huggingface_hub.utils.get_token(), ) + + +def sync( + project: str, space_id: str, private: bool | None = None, force: bool = False +) -> None: + """ + Syncs a local Trackio project's database to a Hugging Face Space. + If the Space does not exist, it will be created. + + Args: + project (`str`): The name of the project to upload. + space_id (`str`): The ID of the Space to upload to (e.g., `"username/space_id"`). + private (`bool`, *optional*): + Whether to make the Space private. If None (default), the repo will be + public unless the organization's default is private. This value is ignored + if the repo already exists. + force (`bool`, *optional*, defaults to `False`): + If `True`, overwrite the existing database without prompting for confirmation. + If `False`, prompt the user before overwriting an existing database. + """ + space_id, _ = preprocess_space_and_dataset_ids(space_id, None) + create_space_if_not_exists(space_id, private=private) + wait_until_space_exists(space_id) + upload_db_to_space(project, space_id, force=force) + print(f"Synced successfully to space: {SPACE_URL.format(space_id=space_id)}") diff --git a/trackio/imports.py b/trackio/imports.py index 3dcc21e..993d32b 100644 --- a/trackio/imports.py +++ b/trackio/imports.py @@ -14,6 +14,7 @@ def import_csv( space_id: str | None = None, dataset_id: str | None = None, private: bool | None = None, + force: bool = False, ) -> None: """ Imports a CSV file into a Trackio project. The CSV file must contain a `"step"` @@ -143,7 +144,7 @@ def import_csv( space_id=space_id, dataset_id=dataset_id, private=private ) deploy.wait_until_space_exists(space_id=space_id) - deploy.upload_db_to_space(project=project, space_id=space_id) + deploy.upload_db_to_space(project=project, space_id=space_id, force=force) print( f"* View dashboard by going to: {deploy.SPACE_URL.format(space_id=space_id)}" ) @@ -156,6 +157,7 @@ def import_tf_events( space_id: str | None = None, dataset_id: str | None = None, private: bool | None = None, + force: bool = False, ) -> None: """ Imports TensorFlow Events files from a directory into a Trackio project. Each @@ -296,7 +298,7 @@ def import_tf_events( space_id, dataset_id=dataset_id, private=private ) deploy.wait_until_space_exists(space_id) - deploy.upload_db_to_space(project, space_id) + deploy.upload_db_to_space(project, space_id, force=force) print( f"* View dashboard by going to: {deploy.SPACE_URL.format(space_id=space_id)}" ) diff --git a/trackio/ui/main.py b/trackio/ui/main.py index b58ddca..68b8057 100644 --- a/trackio/ui/main.py +++ b/trackio/ui/main.py @@ -99,6 +99,18 @@ def get_runs(project) -> list[str]: return SQLiteStorage.get_runs(project) +def upload_db_to_space( + project: str, uploaded_db: gr.FileData, hf_token: str | None +) -> None: + """ + Uploads the database of a local Trackio project to a Hugging Face Space. + """ + fns.check_hf_token_has_write_access(hf_token) + db_project_path = SQLiteStorage.get_project_db_path(project) + os.makedirs(os.path.dirname(db_project_path), exist_ok=True) + shutil.copy(uploaded_db["path"], db_project_path) + + def get_available_metrics(project: str, runs: list[str]) -> list[str]: """Get all available metrics across all runs for x-axis selection.""" if not project or not runs: @@ -285,22 +297,6 @@ def toggle_timer(cb_value): return gr.Timer(active=False) -def upload_db_to_space( - project: str, uploaded_db: gr.FileData, hf_token: str | None -) -> None: - """ - Uploads the database of a local Trackio project to a Hugging Face Space. - """ - fns.check_hf_token_has_write_access(hf_token) - db_project_path = SQLiteStorage.get_project_db_path(project) - if os.path.exists(db_project_path): - raise gr.Error( - f"Trackio database file already exists for project {project}, cannot overwrite." - ) - os.makedirs(os.path.dirname(db_project_path), exist_ok=True) - shutil.copy(uploaded_db["path"], db_project_path) - - def bulk_upload_media(uploads: list[UploadEntry], hf_token: str | None) -> None: """ Uploads media files to a Trackio dashboard. Each entry in the list is a tuple of the project, run, and media file to be uploaded.