From 2573d724f30b3ced0f6883b93fe83a3cc31e9856 Mon Sep 17 00:00:00 2001 From: Uri Shaked Date: Mon, 27 Jan 2025 14:14:24 +0200 Subject: [PATCH] feat: GDS to glTF + S3 upload script --- .gitmodules | 3 ++ README.md | 13 ++++- scripts/.gitignore | 3 ++ scripts/GDS2glTF | 1 + scripts/gltf/.gitignore | 1 + scripts/render_gltf.py | 100 +++++++++++++++++++++++++++++++++++++ scripts/render_projects.py | 27 +++++----- scripts/requirements.txt | 2 + 8 files changed, 137 insertions(+), 13 deletions(-) create mode 100644 .gitmodules create mode 100644 scripts/.gitignore create mode 160000 scripts/GDS2glTF create mode 100644 scripts/gltf/.gitignore create mode 100644 scripts/render_gltf.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..2b8fc66 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "scripts/GDS2glTF"] + path = scripts/GDS2glTF + url = https://github.com/tinytapeout/GDS2glTF diff --git a/README.md b/README.md index 65abaee..c3b2c5b 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,18 @@ Where `` is the identifier of the shuttle (e.g. tt04). You can also specify the scale of the render by passing the `--scale` argument. For example, to render the all the projects in shuttle tt04 at 2x scale, run: ```bash -python render.py tt04 --scale 2 +python render_projects.py tt04 --scale 2 +``` + +## Generating glTF files + +To generate glTF files for the projects, run the following commands: + +```bash +git submodule update --init --recursive +cd scripts +pip install -r requirements.txt -r GDS2glTF/requirements.txt +python render_gltf.py ``` ## License diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000..c7f5b33 --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +.env + diff --git a/scripts/GDS2glTF b/scripts/GDS2glTF new file mode 160000 index 0000000..9706fe1 --- /dev/null +++ b/scripts/GDS2glTF @@ -0,0 +1 @@ +Subproject commit 9706fe1d0680260c90d01c39c2cd505c195b54d4 diff --git a/scripts/gltf/.gitignore b/scripts/gltf/.gitignore new file mode 100644 index 0000000..4a14cf2 --- /dev/null +++ b/scripts/gltf/.gitignore @@ -0,0 +1 @@ +*.gltf diff --git a/scripts/render_gltf.py b/scripts/render_gltf.py new file mode 100644 index 0000000..f71f4dd --- /dev/null +++ b/scripts/render_gltf.py @@ -0,0 +1,100 @@ +""" +SPDX-License-Identifier: Apache-2.0 +Copyright (C) 2025 Tiny Tapeout LTD +Author: Uri Shaked + +Converts all shuttle GDS files to glTF format. Optionally uploads the results to S3. + +To run this script, you need to have the following environment variables set: + +* AWS_ACCESS_KEY_ID - AWS access key +* AWS_SECRET_ACCESS_KEY - AWS secret key + +You can also set the following environment variables: + +* S3_ENDPOINT_URL - S3 endpoint URL (default: s3.amazonaws.com) +* S3_BUCKET - S3 bucket name (default: tt-shuttle-assets) + +You can set these environment variables in a .env file in the same directory as this script. +""" + +import argparse +import json +import logging +import os +import urllib.parse +import urllib.request +from pathlib import Path + +import boto3 +from dotenv import load_dotenv + +from GDS2glTF.gds2gltf import gds2gltf +from render_projects import download_gds + +SCRIPT_DIR = Path(__file__).parent + + +def main(shuttle_id: str, upload_bucket=None): + project_list_url = f"https://index.tinytapeout.com/{shuttle_id}.json" + req = urllib.request.Request( + project_list_url, headers={"User-Agent": "Mozilla/5.0"} + ) + with urllib.request.urlopen(req) as req: + project_list = json.load(req)["projects"] + logging.info(f"Found {len(project_list)} projects in shuttle {shuttle_id}") + + pdk = "sg13g2" if shuttle_id.startswith("ttihp") else "sky130A" + + for project in project_list: + macro = project["macro"] + gds_file = download_gds(shuttle_id, macro) + + logging.info(f"Converting {macro}") + gltf_file = SCRIPT_DIR / "gltf" / shuttle_id / (macro + ".gds.gltf") + gltf_file.parent.mkdir(parents=True, exist_ok=True) + + logging.info(f"Writing {gltf_file}") + gds2gltf(gds_file, gltf_file, pdk_name=pdk) + + if upload_bucket: + logging.info("Uploading to S3...") + with open(gltf_file, "rb") as f: + upload_bucket.put_object( + Key=f"{shuttle_id}/{macro}/{macro}.gds.gltf", + Body=f, + ) + + +if __name__ == "__main__": + load_dotenv() + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser(description="Update shuttle index") + parser.add_argument("shuttle_id", type=str, help="Shuttle ID") + parser.add_argument( + "-u", + "--upload", + action="store_true", + help="Upload to S3/R2", + ) + parser.add_argument( + "--s3-endpoint", + type=str, + help="S3 endpoint", + default=os.getenv("S3_ENDPOINT_URL", "s3.amazonaws.com"), + ) + parser.add_argument( + "--s3-bucket", + type=str, + help="S3 bucket", + default=os.getenv("S3_BUCKET", "tt-shuttle-assets"), + ) + args = parser.parse_args() + + upload_bucket = None + if args.upload: + s3 = boto3.resource("s3", endpoint_url=args.s3_endpoint) + upload_bucket = s3.Bucket(args.s3_bucket) + + main(args.shuttle_id, upload_bucket=upload_bucket) diff --git a/scripts/render_projects.py b/scripts/render_projects.py index 2ffbb3f..7c51d63 100644 --- a/scripts/render_projects.py +++ b/scripts/render_projects.py @@ -11,6 +11,7 @@ import urllib.request from pathlib import Path +from klayout.db import Layout, SaveLayoutOptions from klayout.lay import LayoutView SCRIPT_DIR = Path(__file__).parent @@ -50,7 +51,8 @@ def download_gds(shuttle_id: str, macro: str) -> Path: sys.exit(1) gds_url = shuttle["project_gds_url_template"].format(macro=macro) - if macro == "tt_um_chip_rom": + is_rom = macro == "tt_um_chip_rom" + if is_rom: gds_url = shuttle["gds_url"] logging.info(f"Downloading GDS file from {gds_url}") @@ -64,6 +66,18 @@ def download_gds(shuttle_id: str, macro: str) -> Path: else: f.write(response.read()) + if is_rom: + layout = Layout() + layout.read(target_path) + for cell in layout.each_cell(): + if cell.name == "tt_um_chip_rom" or cell.name.endswith("_tt_um_chip_rom"): + rom_cell = cell + if not rom_cell: + raise Exception("ROM cell not found") + save_options = SaveLayoutOptions() + save_options.add_cell(rom_cell.cell_index()) + layout.write(target_path, options=save_options) + return target_path @@ -72,7 +86,6 @@ def render_gds( output_path: str, pdk: str, scale: float = 1.0, - is_rom=False, ): BOUNDARY_LAYER = TECHNOLOGIES[pdk]["boundary"] hide_layers = TECHNOLOGIES[pdk]["hide_layers"] @@ -82,15 +95,6 @@ def render_gds( lv.max_hier() lv.load_layer_props(SCRIPT_DIR / "lyp" / f"{pdk}.lyp") - if is_rom: - layout = lv.cellview(0).layout() - for cell in layout.each_cell(): - if cell.name == "tt_um_chip_rom" or cell.name.endswith("_tt_um_chip_rom"): - rom_cell = cell - if not rom_cell: - raise Exception("ROM cell not found") - lv.cellview(0).set_cell_name(rom_cell.name) - lv.set_config("background-color", "#ffffff") lv.set_config("grid-visible", "false") lv.set_config("text-visible", "false") @@ -142,7 +146,6 @@ def main(shuttle_id: str, scale: float = 1.0): png_dir / "render.png", pdk=pdk, scale=scale, - is_rom=project["macro"] == "tt_um_chip_rom", ) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 9b02710..a3794a7 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,2 +1,4 @@ klayout==0.29.11 pre-commit==3.4.0 +python-dotenv==1.0.1 +boto3==1.36.6