Randomization Snippets#

Examples of randomization using USD and Isaac Sim APIs. These examples demonstrate how to randomize scenes for synthetic data generation (SDG) in scenarios where default replicator randomizers are not sufficient or applicable.

The snippets are designed to align with the structure and function names used in the replicator example snippets. In comparison they also have the option to write the data to disk by stetting write_data=True.

Prerequisites:

  • Familiarity with USD.

  • Ability to execute code from the Script Editor.

  • Understanding basic replicator concepts, such as subframes.

Randomizing Light Sources#

This snippet sets up a new environment containing a cube and a sphere. It then spawns a given number of lights and randomizes selected attributes for these lights over a specified number of frames.

../_images/isaac_tutorial_replicator_randomization_lights.gif
Randomizing Light Sources
 1import asyncio
 2import os
 3
 4import numpy as np
 5import omni.kit.commands
 6import omni.replicator.core as rep
 7import omni.usd
 8from isaacsim.core.utils.semantics import add_labels
 9from pxr import Gf, Sdf, UsdGeom
10
11omni.usd.get_context().new_stage()
12stage = omni.usd.get_context().get_stage()
13
14sphere = stage.DefinePrim("/World/Sphere", "Sphere")
15UsdGeom.Xformable(sphere).AddTranslateOp().Set((0.0, 1.0, 1.0))
16add_labels(sphere, labels=["sphere"], instance_name="class")
17
18cube = stage.DefinePrim("/World/Cube", "Cube")
19UsdGeom.Xformable(cube).AddTranslateOp().Set((0.0, -2.0, 2.0))
20add_labels(cube, labels=["cube"], instance_name="class")
21
22plane_path = "/World/Plane"
23omni.kit.commands.execute("CreateMeshPrimWithDefaultXform", prim_path=plane_path, prim_type="Plane")
24plane_prim = stage.GetPrimAtPath(plane_path)
25plane_prim.CreateAttribute("xformOp:scale", Sdf.ValueTypeNames.Double3, False).Set(Gf.Vec3d(10, 10, 1))
26
27def sphere_lights(num):
28    lights = []
29    for i in range(num):
30        # "CylinderLight", "DiskLight", "DistantLight", "DomeLight", "RectLight", "SphereLight"
31        prim_type = "SphereLight"
32        next_free_path = omni.usd.get_stage_next_free_path(stage, f"/World/{prim_type}", False)
33        light_prim = stage.DefinePrim(next_free_path, prim_type)
34        UsdGeom.Xformable(light_prim).AddTranslateOp().Set((0.0, 0.0, 0.0))
35        UsdGeom.Xformable(light_prim).AddRotateXYZOp().Set((0.0, 0.0, 0.0))
36        UsdGeom.Xformable(light_prim).AddScaleOp().Set((1.0, 1.0, 1.0))
37        light_prim.CreateAttribute("inputs:enableColorTemperature", Sdf.ValueTypeNames.Bool).Set(True)
38        light_prim.CreateAttribute("inputs:colorTemperature", Sdf.ValueTypeNames.Float).Set(6500.0)
39        light_prim.CreateAttribute("inputs:radius", Sdf.ValueTypeNames.Float).Set(0.5)
40        light_prim.CreateAttribute("inputs:intensity", Sdf.ValueTypeNames.Float).Set(30000.0)
41        light_prim.CreateAttribute("inputs:color", Sdf.ValueTypeNames.Color3f).Set((1.0, 1.0, 1.0))
42        light_prim.CreateAttribute("inputs:exposure", Sdf.ValueTypeNames.Float).Set(0.0)
43        light_prim.CreateAttribute("inputs:diffuse", Sdf.ValueTypeNames.Float).Set(1.0)
44        light_prim.CreateAttribute("inputs:specular", Sdf.ValueTypeNames.Float).Set(1.0)
45        lights.append(light_prim)
46    return lights
47
48async def run_randomizations_async(num_frames, lights, write_data, delay=None):
49    if write_data:
50        out_dir = os.path.join(os.getcwd(), "_out_rand_lights")
51        print(f"Writing data to {out_dir}..")
52        backend = rep.backends.get("DiskBackend")
53        backend.initialize(output_dir=out_dir)
54        writer = rep.WriterRegistry.get("BasicWriter")
55        writer.initialize(backend=backend, rgb=True)
56        cam = rep.functional.create.camera(position=(5, 5, 5), look_at=(0, 0, 0), name="Camera")
57        rp = rep.create.render_product(cam, resolution=(512, 512))
58        writer.attach(rp)
59
60    for _ in range(num_frames):
61        for light in lights:
62            light.GetAttribute("xformOp:translate").Set(
63                (np.random.uniform(-5, 5), np.random.uniform(-5, 5), np.random.uniform(4, 6))
64            )
65            scale_rand = np.random.uniform(0.5, 1.5)
66            light.GetAttribute("xformOp:scale").Set((scale_rand, scale_rand, scale_rand))
67            light.GetAttribute("inputs:colorTemperature").Set(np.random.normal(4500, 1500))
68            light.GetAttribute("inputs:intensity").Set(np.random.normal(25000, 5000))
69            light.GetAttribute("inputs:color").Set(
70                (np.random.uniform(0.1, 0.9), np.random.uniform(0.1, 0.9), np.random.uniform(0.1, 0.9))
71            )
72
73        if write_data:
74            await rep.orchestrator.step_async(rt_subframes=16)
75        else:
76            await omni.kit.app.get_app().next_update_async()
77        # Optional delay between frames to better visualize the randomization in the viewport
78        if delay is not None and delay > 0:
79            await asyncio.sleep(delay)
80
81    # Wait for the data to be written to disk and cleanup writer and render product
82    if write_data:
83        await rep.orchestrator.wait_until_complete_async()
84        writer.detach()
85        rp.destroy()
86
87num_frames = 10
88lights = sphere_lights(10)
89asyncio.ensure_future(run_randomizations_async(num_frames=num_frames, lights=lights, write_data=True, delay=0.2))

Randomizing Textures#

The snippet sets up an environment, spawns a given number of cubes and spheres, and randomizes their textures for the given number of frames. After the randomizations their original materials are reassigned. The snippet also showcases how to create a new material and assign it to a prim.

../_images/isaac_tutorial_replicator_randomization_textures.gif
Randomizing Textures
  1import asyncio
  2import os
  3
  4import numpy as np
  5import omni.replicator.core as rep
  6import omni.usd
  7from isaacsim.core.utils.semantics import add_labels, get_labels
  8from isaacsim.storage.native import get_assets_root_path_async
  9from pxr import Gf, Sdf, UsdGeom, UsdShade
 10
 11omni.usd.get_context().new_stage()
 12stage = omni.usd.get_context().get_stage()
 13dome_light = stage.DefinePrim("/World/DomeLight", "DomeLight")
 14dome_light.CreateAttribute("inputs:intensity", Sdf.ValueTypeNames.Float).Set(1000.0)
 15
 16sphere = stage.DefinePrim("/World/Sphere", "Sphere")
 17UsdGeom.Xformable(sphere).AddTranslateOp().Set((0.0, 0.0, 1.0))
 18add_labels(sphere, labels=["sphere"], instance_name="class")
 19
 20num_cubes = 10
 21for _ in range(num_cubes):
 22    prim_type = "Cube"
 23    next_free_path = omni.usd.get_stage_next_free_path(stage, f"/World/{prim_type}", False)
 24    cube = stage.DefinePrim(next_free_path, prim_type)
 25    UsdGeom.Xformable(cube).AddTranslateOp().Set(
 26        (np.random.uniform(-3.5, 3.5), np.random.uniform(-3.5, 3.5), 1)
 27    )
 28    scale_rand = np.random.uniform(0.25, 0.5)
 29    UsdGeom.Xformable(cube).AddScaleOp().Set((scale_rand, scale_rand, scale_rand))
 30    add_labels(cube, labels=["cube"], instance_name="class")
 31
 32plane_path = "/World/Plane"
 33omni.kit.commands.execute("CreateMeshPrimWithDefaultXform", prim_path=plane_path, prim_type="Plane")
 34plane_prim = stage.GetPrimAtPath(plane_path)
 35plane_prim.CreateAttribute("xformOp:scale", Sdf.ValueTypeNames.Double3, False).Set(Gf.Vec3d(10, 10, 1))
 36
 37def get_shapes():
 38    stage = omni.usd.get_context().get_stage()
 39    shapes = []
 40    for prim in stage.Traverse():
 41        labels = get_labels(prim)
 42        if class_labels := labels.get("class"):
 43            if "cube" in class_labels or "sphere" in class_labels:
 44                shapes.append(prim)
 45    return shapes
 46
 47shapes = get_shapes()
 48
 49def create_omnipbr_material(mtl_url, mtl_name, mtl_path):
 50    stage = omni.usd.get_context().get_stage()
 51    omni.kit.commands.execute("CreateMdlMaterialPrim", mtl_url=mtl_url, mtl_name=mtl_name, mtl_path=mtl_path)
 52    material_prim = stage.GetPrimAtPath(mtl_path)
 53    shader = UsdShade.Shader(omni.usd.get_shader_from_material(material_prim, get_prim=True))
 54
 55    # Add value inputs
 56    shader.CreateInput("diffuse_color_constant", Sdf.ValueTypeNames.Color3f)
 57    shader.CreateInput("reflection_roughness_constant", Sdf.ValueTypeNames.Float)
 58    shader.CreateInput("metallic_constant", Sdf.ValueTypeNames.Float)
 59
 60    # Add texture inputs
 61    shader.CreateInput("diffuse_texture", Sdf.ValueTypeNames.Asset)
 62    shader.CreateInput("reflectionroughness_texture", Sdf.ValueTypeNames.Asset)
 63    shader.CreateInput("metallic_texture", Sdf.ValueTypeNames.Asset)
 64
 65    # Add other attributes
 66    shader.CreateInput("project_uvw", Sdf.ValueTypeNames.Bool)
 67
 68    # Add texture scale and rotate
 69    shader.CreateInput("texture_scale", Sdf.ValueTypeNames.Float2)
 70    shader.CreateInput("texture_rotate", Sdf.ValueTypeNames.Float)
 71
 72    material = UsdShade.Material(material_prim)
 73    return material
 74
 75def create_materials(num):
 76    MDL = "OmniPBR.mdl"
 77    mtl_name, _ = os.path.splitext(MDL)
 78    MAT_PATH = "/World/Looks"
 79    materials = []
 80    for _ in range(num):
 81        prim_path = omni.usd.get_stage_next_free_path(stage, f"{MAT_PATH}/{mtl_name}", False)
 82        mat = create_omnipbr_material(mtl_url=MDL, mtl_name=mtl_name, mtl_path=prim_path)
 83        materials.append(mat)
 84    return materials
 85
 86materials = create_materials(len(shapes))
 87
 88async def run_randomizations_async(num_frames, materials, write_data, delay=None):
 89    assets_root_path = await get_assets_root_path_async()
 90    textures = [
 91        assets_root_path + "/NVIDIA/Materials/vMaterials_2/Ground/textures/aggregate_exposed_diff.jpg",
 92        assets_root_path + "/NVIDIA/Materials/vMaterials_2/Ground/textures/gravel_track_ballast_diff.jpg",
 93        assets_root_path
 94        + "/NVIDIA/Materials/vMaterials_2/Ground/textures/gravel_track_ballast_multi_R_rough_G_ao.jpg",
 95        assets_root_path + "/NVIDIA/Materials/vMaterials_2/Ground/textures/rough_gravel_rough.jpg",
 96    ]
 97
 98    if write_data:
 99        out_dir = os.path.join(os.getcwd(), "_out_rand_textures")
100        print(f"Writing data to {out_dir}..")
101        backend = rep.backends.get("DiskBackend")
102        backend.initialize(output_dir=out_dir)
103        writer = rep.WriterRegistry.get("BasicWriter")
104        writer.initialize(backend=backend, rgb=True)
105        cam = rep.functional.create.camera(position=(5, 5, 5), look_at=(0, 0, 0), name="Camera")
106        rp = rep.create.render_product(cam, resolution=(512, 512))
107        writer.attach(rp)
108
109    # Apply the new materials and store the initial ones to reassign later
110    initial_materials = {}
111    for i, shape in enumerate(shapes):
112        cur_mat, _ = UsdShade.MaterialBindingAPI(shape).ComputeBoundMaterial()
113        initial_materials[shape] = cur_mat
114        UsdShade.MaterialBindingAPI(shape).Bind(materials[i], UsdShade.Tokens.strongerThanDescendants)
115
116    for _ in range(num_frames):
117        for mat in materials:
118            shader = UsdShade.Shader(omni.usd.get_shader_from_material(mat, get_prim=True))
119            diffuse_texture = np.random.choice(textures)
120            shader.GetInput("diffuse_texture").Set(diffuse_texture)
121            project_uvw = np.random.choice([True, False], p=[0.9, 0.1])
122            shader.GetInput("project_uvw").Set(bool(project_uvw))
123            texture_scale = np.random.uniform(0.1, 1)
124            shader.GetInput("texture_scale").Set((texture_scale, texture_scale))
125            texture_rotate = np.random.uniform(0, 45)
126            shader.GetInput("texture_rotate").Set(texture_rotate)
127
128        if write_data:
129            await rep.orchestrator.step_async(rt_subframes=4)
130        else:
131            await omni.kit.app.get_app().next_update_async()
132
133        # Optional delay between frames to better visualize the randomization in the viewport
134        if delay is not None and delay > 0:
135            await asyncio.sleep(delay)
136
137    # Wait for the data to be written to disk and cleanup writer and render product
138    if write_data:
139        await rep.orchestrator.wait_until_complete_async()
140        writer.detach()
141        rp.destroy()
142
143    # Reassign the initial materials
144    for shape, mat in initial_materials.items():
145        if mat:
146            UsdShade.MaterialBindingAPI(shape).Bind(mat, UsdShade.Tokens.strongerThanDescendants)
147        else:
148            UsdShade.MaterialBindingAPI(shape).UnbindAllBindings()
149
150num_frames = 10
151asyncio.ensure_future(run_randomizations_async(num_frames, materials, write_data=True, delay=0.2))

Sequential Randomizations#

The snippet provides an example of more complex randomizations, where the results of the first randomization are used to determine the next randomization. It uses a custom sampler function to set the location of the camera by iterating over (almost) equidistant points on a sphere. The snippet starts by setting up the environment, a forklift, a pallet, a bin, and a dome light. For every randomization frame, it cycles through the dome light textures, moves the pallet to a random location, and then moves the bin so that it is fully on top of the pallet. Finally, it moves the camera to a new location on the sphere, ensuring it faces the bin.

../_images/isaac_tutorial_replicator_randomization_chained_persp.gif ../_images/isaac_tutorial_replicator_randomization_chained_sphere.gif
Sequential Randomizations
  1import asyncio
  2import itertools
  3import os
  4
  5import numpy as np
  6import omni.replicator.core as rep
  7import omni.usd
  8from isaacsim.storage.native import get_assets_root_path_async
  9from pxr import Gf, Usd, UsdGeom, UsdLux
 10
 11
 12# Fibonacci sphere algorithm: https://arxiv.org/pdf/0912.4540
 13def next_point_on_sphere(idx, num_points, radius=1, origin=(0, 0, 0)):
 14    offset = 2.0 / num_points
 15    inc = np.pi * (3.0 - np.sqrt(5.0))
 16    z = ((idx * offset) - 1) + (offset / 2)
 17    phi = ((idx + 1) % num_points) * inc
 18    r = np.sqrt(1 - pow(z, 2))
 19    y = np.cos(phi) * r
 20    x = np.sin(phi) * r
 21    return [(x * radius) + origin[0], (y * radius) + origin[1], (z * radius) + origin[2]]
 22
 23
 24async def run_randomizations_async(
 25    num_frames, forklift_path, pallet_path, bin_path, dome_textures, write_data, delay=None
 26):
 27    assets_root_path = await get_assets_root_path_async()
 28
 29    await omni.usd.get_context().new_stage_async()
 30    stage = omni.usd.get_context().get_stage()
 31
 32    dome_light = UsdLux.DomeLight.Define(stage, "/World/Lights/DomeLight")
 33    dome_light.GetIntensityAttr().Set(1000)
 34
 35    forklift_prim = stage.DefinePrim("/World/Forklift", "Xform")
 36    forklift_prim.GetReferences().AddReference(assets_root_path + forklift_path)
 37    if not forklift_prim.GetAttribute("xformOp:translate"):
 38        UsdGeom.Xformable(forklift_prim).AddTranslateOp()
 39    forklift_prim.GetAttribute("xformOp:translate").Set((-4.5, -4.5, 0))
 40
 41    pallet_prim = stage.DefinePrim("/World/Pallet", "Xform")
 42    pallet_prim.GetReferences().AddReference(assets_root_path + pallet_path)
 43    if not pallet_prim.GetAttribute("xformOp:translate"):
 44        UsdGeom.Xformable(pallet_prim).AddTranslateOp()
 45    if not pallet_prim.GetAttribute("xformOp:rotateXYZ"):
 46        UsdGeom.Xformable(pallet_prim).AddRotateXYZOp()
 47
 48    bin_prim = stage.DefinePrim("/World/Bin", "Xform")
 49    bin_prim.GetReferences().AddReference(assets_root_path + bin_path)
 50    if not bin_prim.GetAttribute("xformOp:translate"):
 51        UsdGeom.Xformable(bin_prim).AddTranslateOp()
 52    if not bin_prim.GetAttribute("xformOp:rotateXYZ"):
 53        UsdGeom.Xformable(bin_prim).AddRotateXYZOp()
 54
 55    view_cam = stage.DefinePrim("/World/Camera", "Camera")
 56    if not view_cam.GetAttribute("xformOp:translate"):
 57        UsdGeom.Xformable(view_cam).AddTranslateOp()
 58    if not view_cam.GetAttribute("xformOp:orient"):
 59        UsdGeom.Xformable(view_cam).AddOrientOp()
 60
 61    dome_textures_full = [assets_root_path + tex for tex in dome_textures]
 62    textures_cycle = itertools.cycle(dome_textures_full)
 63
 64    if write_data:
 65        out_dir = os.path.join(os.getcwd(), "_out_rand_sphere_scan")
 66        print(f"Writing data to {out_dir}..")
 67        backend = rep.backends.get("DiskBackend")
 68        backend.initialize(output_dir=out_dir)
 69        writer = rep.WriterRegistry.get("BasicWriter")
 70        writer.initialize(backend=backend, rgb=True)
 71        persp_cam = rep.functional.create.camera(position=(5, 5, 5), look_at=(0, 0, 0), name="PerspCamera")
 72        rp_persp = rep.create.render_product(persp_cam, (512, 512), name="PerspView")
 73        rp_view = rep.create.render_product(view_cam, (512, 512), name="SphereView")
 74        writer.attach([rp_view, rp_persp])
 75
 76    bb_cache = UsdGeom.BBoxCache(time=Usd.TimeCode.Default(), includedPurposes=[UsdGeom.Tokens.default_])
 77    pallet_size = bb_cache.ComputeWorldBound(pallet_prim).GetRange().GetSize()
 78    pallet_length = pallet_size.GetLength()
 79    bin_size = bb_cache.ComputeWorldBound(bin_prim).GetRange().GetSize()
 80
 81    for i in range(num_frames):
 82        # Set next background texture every nth frame and run an app update
 83        if i % 5 == 0:
 84            dome_light.GetTextureFileAttr().Set(next(textures_cycle))
 85            await omni.kit.app.get_app().next_update_async()
 86
 87        # Randomize pallet pose
 88        pallet_prim.GetAttribute("xformOp:translate").Set(
 89            Gf.Vec3d(np.random.uniform(-1.5, 1.5), np.random.uniform(-1.5, 1.5), 0)
 90        )
 91        rand_z_rot = np.random.uniform(-90, 90)
 92        pallet_prim.GetAttribute("xformOp:rotateXYZ").Set(Gf.Vec3d(0, 0, rand_z_rot))
 93        pallet_tf_mat = omni.usd.get_world_transform_matrix(pallet_prim)
 94        pallet_rot = pallet_tf_mat.ExtractRotation()
 95        pallet_pos = pallet_tf_mat.ExtractTranslation()
 96
 97        # Randomize bin position on top of the rotated pallet area making sure the bin is fully on the pallet
 98        rand_transl_x = np.random.uniform(-pallet_size[0] / 2 + bin_size[0] / 2, pallet_size[0] / 2 - bin_size[0] / 2)
 99        rand_transl_y = np.random.uniform(-pallet_size[1] / 2 + bin_size[1] / 2, pallet_size[1] / 2 - bin_size[1] / 2)
100
101        # Adjust bin position to account for the random rotation of the pallet
102        rand_z_rot_rad = np.deg2rad(rand_z_rot)
103        rot_adjusted_transl_x = rand_transl_x * np.cos(rand_z_rot_rad) - rand_transl_y * np.sin(rand_z_rot_rad)
104        rot_adjusted_transl_y = rand_transl_x * np.sin(rand_z_rot_rad) + rand_transl_y * np.cos(rand_z_rot_rad)
105        bin_prim.GetAttribute("xformOp:translate").Set(
106            Gf.Vec3d(
107                pallet_pos[0] + rot_adjusted_transl_x,
108                pallet_pos[1] + rot_adjusted_transl_y,
109                pallet_pos[2] + pallet_size[2] + bin_size[2] / 2,
110            )
111        )
112        # Keep bin rotation aligned with pallet
113        bin_prim.GetAttribute("xformOp:rotateXYZ").Set(pallet_rot.GetAxis() * pallet_rot.GetAngle())
114
115        # Get next camera position on a sphere looking at the bin with a randomized distance
116        rand_radius = np.random.normal(3, 0.5) * pallet_length
117        bin_pos = omni.usd.get_world_transform_matrix(bin_prim).ExtractTranslation()
118        cam_pos = next_point_on_sphere(i, num_points=num_frames, radius=rand_radius, origin=bin_pos)
119        view_cam.GetAttribute("xformOp:translate").Set(Gf.Vec3d(*cam_pos))
120
121        eye = Gf.Vec3d(*cam_pos)
122        target = Gf.Vec3d(*bin_pos)
123        up_axis = Gf.Vec3d(0, 0, 1)
124        look_at_quatd = Gf.Matrix4d().SetLookAt(eye, target, up_axis).GetInverse().ExtractRotation().GetQuat()
125        view_cam.GetAttribute("xformOp:orient").Set(Gf.Quatf(look_at_quatd))
126
127        if write_data:
128            await rep.orchestrator.step_async(rt_subframes=4, delta_time=0.0)
129        else:
130            await omni.kit.app.get_app().next_update_async()
131        # Optional delay between frames to better visualize the randomization in the viewport
132        if delay is not None and delay > 0:
133            await asyncio.sleep(delay)
134
135    # Wait for the data to be written to disk and cleanup writer and render products
136    if write_data:
137        await rep.orchestrator.wait_until_complete_async()
138        writer.detach()
139        rp_persp.destroy()
140        rp_view.destroy()
141
142
143NUM_FRAMES = 90
144FORKLIFT_PATH = "/Isaac/Props/Forklift/forklift.usd"
145PALLET_PATH = "/Isaac/Props/Pallet/pallet.usd"
146BIN_PATH = "/Isaac/Props/KLT_Bin/small_KLT_visual.usd"
147DOME_TEXTURES = [
148    "/NVIDIA/Assets/Skies/Cloudy/champagne_castle_1_4k.hdr",
149    "/NVIDIA/Assets/Skies/Clear/evening_road_01_4k.hdr",
150    "/NVIDIA/Assets/Skies/Clear/mealie_road_4k.hdr",
151    "/NVIDIA/Assets/Skies/Clear/qwantani_4k.hdr",
152]
153asyncio.ensure_future(
154    run_randomizations_async(
155        NUM_FRAMES, FORKLIFT_PATH, PALLET_PATH, BIN_PATH, DOME_TEXTURES, write_data=True, delay=0.2
156    )
157)

Physics-based Randomized Volume Filling#

The snippet randomizes the stacking of objects on multiple surfaces. It randomly spawns a given number of pallets in the selected areas and then spawns physically simulated boxes on top of them. A temporary collision box area is created around the pallets to prevent the boxes from falling off. After all the boxes have been dropped, they are moved in various directions and finally pulled towards the center of the pallet for more stable stacking. Finally, the collision area is removed, after which the boxes can also fall to the ground. To allow easier sliding of the boxes into more stable positions, their friction is temporarily reduced during the simulation.

../_images/isaac_tutorial_replicator_randomization_volume_fill.gif ../_images/isaac_tutorial_replicator_randomization_volume_fill_warehouse.gif
Physics-based Randomized Volume Filling
  1import asyncio
  2import os
  3import random
  4from itertools import chain
  5
  6import carb
  7import omni.kit.app
  8import omni.physics.core
  9import omni.replicator.core as rep
 10import omni.usd
 11from isaacsim.storage.native import get_assets_root_path_async
 12from pxr import Gf, PhysicsSchemaTools, PhysxSchema, Sdf, Usd, UsdGeom, UsdPhysics, UsdShade, UsdUtils
 13
 14# Add transformation properties to the prim (if not already present)
 15def set_transform_attributes(prim, location=None, orientation=None, rotation=None, scale=None):
 16    if location is not None:
 17        if not prim.HasAttribute("xformOp:translate"):
 18            UsdGeom.Xformable(prim).AddTranslateOp()
 19        prim.GetAttribute("xformOp:translate").Set(location)
 20    if orientation is not None:
 21        if not prim.HasAttribute("xformOp:orient"):
 22            UsdGeom.Xformable(prim).AddOrientOp()
 23        prim.GetAttribute("xformOp:orient").Set(orientation)
 24    if rotation is not None:
 25        if not prim.HasAttribute("xformOp:rotateXYZ"):
 26            UsdGeom.Xformable(prim).AddRotateXYZOp()
 27        prim.GetAttribute("xformOp:rotateXYZ").Set(rotation)
 28    if scale is not None:
 29        if not prim.HasAttribute("xformOp:scale"):
 30            UsdGeom.Xformable(prim).AddScaleOp()
 31        prim.GetAttribute("xformOp:scale").Set(scale)
 32
 33# Enables collisions with the asset (without rigid body dynamics the asset will be static)
 34def add_colliders(prim):
 35    # Iterate descendant prims (including root) and add colliders to mesh or primitive types
 36    for desc_prim in Usd.PrimRange(prim):
 37        if desc_prim.IsA(UsdGeom.Mesh) or desc_prim.IsA(UsdGeom.Gprim):
 38            # Physics
 39            if not desc_prim.HasAPI(UsdPhysics.CollisionAPI):
 40                collision_api = UsdPhysics.CollisionAPI.Apply(desc_prim)
 41            else:
 42                collision_api = UsdPhysics.CollisionAPI(desc_prim)
 43            collision_api.CreateCollisionEnabledAttr(True)
 44
 45        # Add mesh specific collision properties only to mesh types
 46        if desc_prim.IsA(UsdGeom.Mesh):
 47            if not desc_prim.HasAPI(UsdPhysics.MeshCollisionAPI):
 48                mesh_collision_api = UsdPhysics.MeshCollisionAPI.Apply(desc_prim)
 49            else:
 50                mesh_collision_api = UsdPhysics.MeshCollisionAPI(desc_prim)
 51            mesh_collision_api.CreateApproximationAttr().Set("convexHull")
 52
 53# Enables rigid body dynamics (physics simulation) on the prim (having valid colliders is recommended)
 54def add_rigid_body_dynamics(prim, disable_gravity=False, angular_damping=None):
 55    # Physics
 56    if not prim.HasAPI(UsdPhysics.RigidBodyAPI):
 57        rigid_body_api = UsdPhysics.RigidBodyAPI.Apply(prim)
 58    else:
 59        rigid_body_api = UsdPhysics.RigidBodyAPI(prim)
 60    rigid_body_api.CreateRigidBodyEnabledAttr(True)
 61    # PhysX
 62    if not prim.HasAPI(PhysxSchema.PhysxRigidBodyAPI):
 63        physx_rigid_body_api = PhysxSchema.PhysxRigidBodyAPI.Apply(prim)
 64    else:
 65        physx_rigid_body_api = PhysxSchema.PhysxRigidBodyAPI(prim)
 66    physx_rigid_body_api.GetDisableGravityAttr().Set(disable_gravity)
 67    if angular_damping is not None:
 68        physx_rigid_body_api.CreateAngularDampingAttr().Set(angular_damping)
 69
 70# Create a new prim with the provided asset URL and transform properties
 71def create_asset(stage, asset_url, path, location=None, rotation=None, orientation=None, scale=None):
 72    prim_path = omni.usd.get_stage_next_free_path(stage, path, False)
 73    prim = stage.DefinePrim(prim_path, "Xform")
 74    prim.GetReferences().AddReference(asset_url)
 75    set_transform_attributes(prim, location=location, rotation=rotation, orientation=orientation, scale=scale)
 76    return prim
 77
 78# Create a new prim with the provided asset URL and transform properties including colliders
 79def create_asset_with_colliders(
 80    stage, asset_url, path, location=None, rotation=None, orientation=None, scale=None
 81):
 82    prim = create_asset(stage, asset_url, path, location, rotation, orientation, scale)
 83    add_colliders(prim)
 84    return prim
 85
 86# Create collision walls around the top surface of the prim with the given height and thickness
 87def create_collision_walls(stage, prim, bbox_cache=None, height=2, thickness=0.3, material=None, visible=False):
 88    # Use the untransformed axis-aligned bounding box to calculate the prim surface size and center
 89    if bbox_cache is None:
 90        bbox_cache = UsdGeom.BBoxCache(Usd.TimeCode.Default(), includedPurposes=[UsdGeom.Tokens.default_])
 91    local_range = bbox_cache.ComputeWorldBound(prim).GetRange()
 92    width, depth, local_height = local_range.GetSize()
 93    # Raise the midpoint height to the prim's surface
 94    mid = local_range.GetMidpoint() + Gf.Vec3d(0, 0, local_height / 2)
 95
 96    # Define the walls (name, location, size) with the specified thickness added externally to the surface and height
 97    walls = [
 98        ("floor", (mid[0], mid[1], mid[2] - thickness / 2), (width, depth, thickness)),
 99        ("ceiling", (mid[0], mid[1], mid[2] + height + thickness / 2), (width, depth, thickness)),
100        (
101            "left_wall",
102            (mid[0] - (width + thickness) / 2, mid[1], mid[2] + height / 2),
103            (thickness, depth, height),
104        ),
105        (
106            "right_wall",
107            (mid[0] + (width + thickness) / 2, mid[1], mid[2] + height / 2),
108            (thickness, depth, height),
109        ),
110        (
111            "front_wall",
112            (mid[0], mid[1] + (depth + thickness) / 2, mid[2] + height / 2),
113            (width, thickness, height),
114        ),
115        (
116            "back_wall",
117            (mid[0], mid[1] - (depth + thickness) / 2, mid[2] + height / 2),
118            (width, thickness, height),
119        ),
120    ]
121
122    # Use the parent prim path to create the walls as children (use local coordinates)
123    prim_path = prim.GetPath()
124    collision_walls = []
125    for name, location, size in walls:
126        prim = stage.DefinePrim(f"{prim_path}/{name}", "Cube")
127        scale = (size[0] / 2.0, size[1] / 2.0, size[2] / 2.0)
128        set_transform_attributes(prim, location=location, scale=scale)
129        add_colliders(prim)
130        if not visible:
131            UsdGeom.Imageable(prim).MakeInvisible()
132        if material is not None:
133            mat_binding_api = UsdShade.MaterialBindingAPI.Apply(prim)
134            mat_binding_api.Bind(material, UsdShade.Tokens.weakerThanDescendants, "physics")
135        collision_walls.append(prim)
136    return collision_walls
137
138# Slide the assets independently in perpendicular directions and then pull them all together towards the given center
139async def apply_forces_async(stage, boxes, pallet, strength=550, strength_center_multiplier=2):
140    timeline = omni.timeline.get_timeline_interface()
141    timeline.play()
142    # Get the pallet center and forward vector to apply forces in the perpendicular directions and towards the center
143    pallet_tf: Gf.Matrix4d = UsdGeom.Xformable(pallet).ComputeLocalToWorldTransform(Usd.TimeCode.Default())
144    pallet_center = pallet_tf.ExtractTranslation()
145    pallet_rot: Gf.Rotation = pallet_tf.ExtractRotation()
146    force_forward = Gf.Vec3d(pallet_rot.TransformDir(Gf.Vec3d(1, 0, 0))) * strength
147    force_right = Gf.Vec3d(pallet_rot.TransformDir(Gf.Vec3d(0, 1, 0))) * strength
148
149    physics_simulation_interface = omni.physics.core.get_physics_simulation_interface()
150    stage_id = UsdUtils.StageCache.Get().GetId(stage).ToLongInt()
151    for box_prim in boxes:
152        body_path = PhysicsSchemaTools.sdfPathToInt(box_prim.GetPath())
153        forces = [force_forward, force_right, -force_forward, -force_right]
154        for force in chain(forces, forces):
155            box_tf: Gf.Matrix4d = UsdGeom.Xformable(box_prim).ComputeLocalToWorldTransform(
156                Usd.TimeCode.Default()
157            )
158            box_position = carb.Float3(*box_tf.ExtractTranslation())
159            physics_simulation_interface.add_force_at_pos(
160                stage_id, body_path, carb.Float3(force), box_position, omni.physics.core.ForceMode.FORCE
161            )
162            for _ in range(10):
163                await omni.kit.app.get_app().next_update_async()
164
165    # Pull all box at once to the pallet center
166    for box_prim in boxes:
167        body_path = PhysicsSchemaTools.sdfPathToInt(box_prim.GetPath())
168        box_tf: Gf.Matrix4d = UsdGeom.Xformable(box_prim).ComputeLocalToWorldTransform(Usd.TimeCode.Default())
169        box_location = box_tf.ExtractTranslation()
170        force_to_center = (pallet_center - box_location) * strength * strength_center_multiplier
171        physics_simulation_interface.add_force_at_pos(
172            stage_id,
173            body_path,
174            carb.Float3(*force_to_center),
175            carb.Float3(*box_location),
176            omni.physics.core.ForceMode.FORCE,
177        )
178    for _ in range(20):
179        await omni.kit.app.get_app().next_update_async()
180    timeline.pause()
181
182# Create a new stage and and run the example scenario
183async def stack_boxes_on_pallet_async(
184    pallet_prim, boxes_urls_and_weights, num_boxes, drop_height=1.5, drop_margin=0.2
185):
186    pallet_path = pallet_prim.GetPath()
187    print(f"[BoxStacking] Running scenario for pallet {pallet_path} with {num_boxes} boxes..")
188    stage = omni.usd.get_context().get_stage()
189    bbox_cache = UsdGeom.BBoxCache(Usd.TimeCode.Default(), includedPurposes=[UsdGeom.Tokens.default_])
190
191    # Create a custom physics material to allow the boxes to easily slide into stacking positions
192    material_path = f"{pallet_path}/Looks/PhysicsMaterial"
193    default_material = UsdShade.Material.Define(stage, material_path)
194    physics_material = UsdPhysics.MaterialAPI.Apply(default_material.GetPrim())
195    physics_material.CreateRestitutionAttr().Set(0.0)  # Inelastic collision (no bouncing)
196    physics_material.CreateStaticFrictionAttr().Set(0.01)  # Small friction to allow sliding of stationary boxes
197    physics_material.CreateDynamicFrictionAttr().Set(0.01)  # Small friction to allow sliding of moving boxes
198
199    # Apply the physics material to the pallet
200    mat_binding_api = UsdShade.MaterialBindingAPI.Apply(pallet_prim)
201    mat_binding_api.Bind(default_material, UsdShade.Tokens.weakerThanDescendants, "physics")
202
203    # Create collision walls around the top of the pallet and apply the physics material to them
204    collision_walls = create_collision_walls(
205        stage, pallet_prim, bbox_cache, height=drop_height + drop_margin, material=default_material
206    )
207
208    # Create the random boxes (without physics) with the specified weights and sort them by size (volume)
209    box_urls, box_weights = zip(*boxes_urls_and_weights)
210    rand_boxes_urls = random.choices(box_urls, weights=box_weights, k=num_boxes)
211    boxes = [
212        create_asset(stage, box_url, f"{pallet_path}_Boxes/Box_{i}")
213        for i, box_url in enumerate(rand_boxes_urls)
214    ]
215    boxes.sort(key=lambda box: bbox_cache.ComputeLocalBound(box).GetVolume(), reverse=True)
216
217    # Calculate the drop area above the pallet taking into account the pallet surface, drop height and the margin
218    # Note: The boxes can be spawned colliding with the surrounding collision walls as they will be pushed inwards
219    pallet_range = bbox_cache.ComputeWorldBound(pallet_prim).GetRange()
220    pallet_width, pallet_depth, pallet_heigth = pallet_range.GetSize()
221    # Move the spawn center at the given height above the pallet surface
222    spawn_center = pallet_range.GetMidpoint() + Gf.Vec3d(0, 0, pallet_heigth / 2 + drop_height)
223    spawn_width, spawn_depth = pallet_width / 2 - drop_margin, pallet_depth / 2 - drop_margin
224
225    # Use the pallet local-to-world transform to apply the local random offsets relative to the pallet
226    pallet_tf: Gf.Matrix4d = UsdGeom.Xformable(pallet_prim).ComputeLocalToWorldTransform(Usd.TimeCode.Default())
227    pallet_rot: Gf.Rotation = pallet_tf.ExtractRotation()
228
229    # Simulate dropping the boxes from random poses on the pallet
230    timeline = omni.timeline.get_timeline_interface()
231    for box_prim in boxes:
232        # Create a random location and orientation for the box within the drop area in local frame
233        local_loc = spawn_center + Gf.Vec3d(
234            random.uniform(-spawn_width, spawn_width), random.uniform(-spawn_depth, spawn_depth), 0
235        )
236        axes = [Gf.Vec3d(1, 0, 0), Gf.Vec3d(0, 1, 0), Gf.Vec3d(0, 0, 1)]
237        angles = [random.choice([180, 90, 0, -90, -180]) + random.uniform(-3, 3) for _ in axes]
238        local_rot = Gf.Rotation()
239        for axis, angle in zip(axes, angles):
240            local_rot *= Gf.Rotation(axis, angle)
241
242        # Transform the local pose to the pallet's world coordinate system
243        world_loc = pallet_tf.Transform(local_loc)
244        world_quat = Gf.Quatf((pallet_rot * local_rot).GetQuat())
245
246        # Set the spawn pose and enable collisions and rigid body dynamics with dampened angular movements
247        set_transform_attributes(box_prim, location=world_loc, orientation=world_quat)
248        add_colliders(box_prim)
249        add_rigid_body_dynamics(box_prim, angular_damping=0.9)
250
251        # Bind the physics material to the box (allow frictionless sliding)
252        mat_binding_api = UsdShade.MaterialBindingAPI.Apply(box_prim)
253        mat_binding_api.Bind(default_material, UsdShade.Tokens.weakerThanDescendants, "physics")
254        # Wait for an app update to load the new attributes
255        await omni.kit.app.get_app().next_update_async()
256
257        # Play simulation for a few frames for each box
258        timeline.play()
259        for _ in range(20):
260            await omni.kit.app.get_app().next_update_async()
261        timeline.pause()
262
263    # Iteratively apply forces to the boxes to move them around then pull them all together towards the pallet center
264    await apply_forces_async(stage, boxes, pallet_prim)
265
266    # Remove rigid body dynamics of the boxes until all other scenarios are completed
267    for box in boxes:
268        UsdPhysics.RigidBodyAPI(box).GetRigidBodyEnabledAttr().Set(False)
269
270    # Increase the friction to prevent sliding of the boxes on the pallet before removing the collision walls
271    physics_material.CreateStaticFrictionAttr().Set(0.9)
272    physics_material.CreateDynamicFrictionAttr().Set(0.9)
273
274    # Remove collision walls
275    for wall in collision_walls:
276        stage.RemovePrim(wall.GetPath())
277    return boxes
278
279# Run the example scenario
280async def run_box_stacking_scenarios_async(num_pallets, env_url=None, write_data=False):
281    # Get assets root path once for all asset loading operations
282    assets_root_path = await get_assets_root_path_async()
283
284    # List of pallets and boxes to randomly choose from with their respective weights
285    pallets_urls_and_weights = [
286        (assets_root_path + "/Isaac/Environments/Simple_Warehouse/Props/SM_PaletteA_01.usd", 0.25),
287        (assets_root_path + "/Isaac/Environments/Simple_Warehouse/Props/SM_PaletteA_02.usd", 0.75),
288    ]
289    boxes_urls_and_weights = [
290        (assets_root_path + "/Isaac/Environments/Simple_Warehouse/Props/SM_CardBoxA_01.usd", 0.02),
291        (assets_root_path + "/Isaac/Environments/Simple_Warehouse/Props/SM_CardBoxB_01.usd", 0.06),
292        (assets_root_path + "/Isaac/Environments/Simple_Warehouse/Props/SM_CardBoxC_01.usd", 0.12),
293        (assets_root_path + "/Isaac/Environments/Simple_Warehouse/Props/SM_CardBoxD_01.usd", 0.80),
294    ]
295
296    # Load a predefined or create a new stage
297    if env_url is not None:
298        env_path = env_url if env_url.startswith("omniverse://") else assets_root_path + env_url
299        omni.usd.get_context().open_stage(env_path)
300        stage = omni.usd.get_context().get_stage()
301    else:
302        omni.usd.get_context().new_stage()
303        stage = omni.usd.get_context().get_stage()
304        distant_light = stage.DefinePrim("/World/Lights/DistantLight", "DistantLight")
305        distant_light.CreateAttribute("inputs:intensity", Sdf.ValueTypeNames.Float).Set(400.0)
306        if not distant_light.HasAttribute("xformOp:rotateXYZ"):
307            UsdGeom.Xformable(distant_light).AddRotateXYZOp()
308        distant_light.GetAttribute("xformOp:rotateXYZ").Set((0, 60, 0))
309        dome_light = stage.DefinePrim("/World/Lights/DomeLight", "DomeLight")
310        dome_light.CreateAttribute("inputs:intensity", Sdf.ValueTypeNames.Float).Set(500.0)
311
312    # Spawn the pallets
313    pallets = []
314    pallets_urls, pallets_weights = zip(*pallets_urls_and_weights)
315    rand_pallet_urls = random.choices(pallets_urls, weights=pallets_weights, k=num_pallets)
316    # Custom pallet poses for the evnironment
317    custom_pallet_locations = [
318        (-9.3, 5.3, 1.3),
319        (-9.3, 7.3, 1.3),
320        (-9.3, -0.6, 1.3),
321    ]
322    random.shuffle(custom_pallet_locations)
323    for i, pallet_url in enumerate(rand_pallet_urls):
324        # Use a custom location for every other pallet
325        if env_url is not None:
326            if i % 2 == 0 and custom_pallet_locations:
327                rand_loc = Gf.Vec3d(*custom_pallet_locations.pop())
328            else:
329                rand_loc = Gf.Vec3d(-6.5, i * 1.75, 0) + Gf.Vec3d(
330                    random.uniform(-0.2, 0.2), random.uniform(0, 0.2), 0
331                )
332        else:
333            rand_loc = Gf.Vec3d(i * 1.5, 0, 0) + Gf.Vec3d(random.uniform(0, 0.2), random.uniform(-0.2, 0.2), 0)
334        rand_rot = (0, 0, random.choice([180, 90, 0, -90, -180]) + random.uniform(-15, 15))
335        pallet_prim = create_asset_with_colliders(
336            stage, pallet_url, f"/World/Pallet_{i}", location=rand_loc, rotation=rand_rot
337        )
338        pallets.append(pallet_prim)
339
340    # Stack the boxes on the pallets
341    total_boxes = []
342    for pallet in pallets:
343        if env_url is not None:
344            rand_num_boxes = random.randint(8, 15)
345            stacked_boxes = await stack_boxes_on_pallet_async(
346                pallet, boxes_urls_and_weights, num_boxes=rand_num_boxes, drop_height=1.0
347            )
348        else:
349            rand_num_boxes = random.randint(12, 20)
350            stacked_boxes = await stack_boxes_on_pallet_async(
351                pallet, boxes_urls_and_weights, num_boxes=rand_num_boxes
352            )
353        total_boxes.extend(stacked_boxes)
354
355    # Re-enable rigid body dynamics of the boxes and run the simulation for a while
356    for box in total_boxes:
357        UsdPhysics.RigidBodyAPI(box).GetRigidBodyEnabledAttr().Set(True)
358    timeline = omni.timeline.get_timeline_interface()
359    timeline.play()
360    for _ in range(200):
361        await omni.kit.app.get_app().next_update_async()
362    timeline.pause()
363
364    if write_data:
365        out_dir = os.path.join(os.getcwd(), "_out_box_stacking")
366        print(f"Writing data to {out_dir}..")
367        backend = rep.backends.get("DiskBackend")
368        backend.initialize(output_dir=out_dir)
369        writer = rep.WriterRegistry.get("BasicWriter")
370        writer.initialize(backend=backend, rgb=True)
371        cam = rep.functional.create.camera(position=(5, -5, 2), look_at=(0, 0, 0), name="PalletCamera")
372        rp = rep.create.render_product(cam, resolution=(512, 512))
373        writer.attach(rp)
374
375        # Capture the data and wait for the data to be written to disk
376        await rep.orchestrator.step_async(rt_subframes=8)
377
378        # Wait for the data to be written to disk and cleanup
379        await rep.orchestrator.wait_until_complete_async()
380        writer.detach()
381        rp.destroy()
382
383# asyncio.ensure_future(run_box_stacking_scenarios_async(num_pallets=1, write_data=True))
384asyncio.ensure_future(
385    run_box_stacking_scenarios_async(
386        num_pallets=6, env_url="/Isaac/Environments/Simple_Warehouse/warehouse.usd", write_data=True
387    )
388)

Simready Assets SDG Example#

Script editor example for using SimReady Assets to randomize the scene. SimReady Assets are physically accurate 3D objects with realistic properties, behavior, and data connections that are optimized for simulation.

Note

The example can only run in async mode and requires the SimReady Explorer window to be enabled to process the search requests.

The example script will create an SDG randomization and capture pipeline scenario with a table, a plate, and a number of items on top of the plate. The scene will be simulated for a while and then the captured images will be saved to disk.

The standalone example can also be run directly (on Windows use python.bat instead of python.sh):

./python.sh standalone_examples/api/isaacsim.replicator.examples/simready_assets_sdg.py
../_images/isim_5.0_replicator_tut_viewport_randomization_simready_assets.jpg
Simready Assets SDG Example
  1import asyncio
  2import os
  3import time
  4
  5import carb.settings
  6import numpy as np
  7import omni.kit.app
  8import omni.replicator.core as rep
  9import omni.timeline
 10import omni.usd
 11from isaacsim.core.utils.semantics import upgrade_prim_semantics_to_labels
 12from pxr import Sdf, Usd, UsdGeom, UsdPhysics
 13
 14# Make sure the simready explorer extension is enabled
 15ext_manager = omni.kit.app.get_app().get_extension_manager()
 16if not ext_manager.is_extension_enabled("omni.simready.explorer"):
 17    ext_manager.set_extension_enabled_immediate("omni.simready.explorer", True)
 18import omni.simready.explorer as sre
 19
 20
 21def enable_simready_explorer() -> None:
 22    """Enable the SimReady Explorer window if not already open."""
 23    if sre.get_instance().browser_model is None:
 24        import omni.kit.actions.core as actions
 25
 26        actions.execute_action("omni.simready.explorer", "toggle_window")
 27
 28
 29def set_prim_variants(prim: Usd.Prim, variants: dict[str, str]) -> None:
 30    """Set variant selections on a prim from a dictionary of variant set names to values."""
 31    vsets = prim.GetVariantSets()
 32    for name, value in variants.items():
 33        vset = vsets.GetVariantSet(name)
 34        if vset:
 35            vset.SetVariantSelection(value)
 36
 37
 38async def search_assets_async() -> tuple[list, list, list]:
 39    """Search for SimReady assets (tables, dishes, items) asynchronously."""
 40    print(f"[SDG] Searching for SimReady assets...")
 41    start_time = time.time()
 42    tables = await sre.find_assets(["table", "furniture"])
 43    print(f"[SDG]   - Found {len(tables)} tables ({time.time() - start_time:.2f}s)")
 44    start_time = time.time()
 45    plates = await sre.find_assets(["plate"])
 46    print(f"[SDG]   - Found {len(plates)} plates ({time.time() - start_time:.2f}s)")
 47    start_time = time.time()
 48    bowls = await sre.find_assets(["bowl"])
 49    print(f"[SDG]   - Found {len(bowls)} bowls ({time.time() - start_time:.2f}s)")
 50    dishes = plates + bowls
 51    start_time = time.time()
 52    fruits = await sre.find_assets(["fruit"])
 53    print(f"[SDG]   - Found {len(fruits)} fruits ({time.time() - start_time:.2f}s)")
 54    start_time = time.time()
 55    vegetables = await sre.find_assets(["vegetable"])
 56    print(f"[SDG]   - Found {len(vegetables)} vegetables ({time.time() - start_time:.2f}s)")
 57    items = fruits + vegetables
 58    return tables, dishes, items
 59
 60
 61async def run_simready_randomization_async(
 62    stage: Usd.Stage,
 63    camera_prim: Usd.Prim,
 64    render_product,
 65    tables: list,
 66    dishes: list,
 67    items: list,
 68    rng: np.random.Generator = None,
 69) -> None:
 70    """Randomize a scene with SimReady assets, run physics, and capture the result."""
 71    if rng is None:
 72        rng = np.random.default_rng()
 73
 74    print(f"[SDG]   Creating anonymous variation layer for the randomizations...")
 75    root_layer = stage.GetRootLayer()
 76    variation_layer = Sdf.Layer.CreateAnonymous("variation")
 77    root_layer.subLayerPaths.insert(0, variation_layer.identifier)
 78    stage.SetEditTarget(variation_layer)
 79
 80    # Load the simready assets with rigid body properties
 81    variants = {"PhysicsVariant": "RigidBody"}
 82    rep.functional.create.scope(name="Assets")
 83
 84    # Choose a random table and add it to the stage
 85    print(f"[SDG]   Loading assets...")
 86    table_asset = tables[rng.integers(len(tables))]
 87    start_time = time.time()
 88    table_prim = rep.functional.create.reference(usd_path=table_asset.main_url, parent="/Assets", name=table_asset.name)
 89    set_prim_variants(table_prim, variants)
 90    upgrade_prim_semantics_to_labels(table_prim)
 91    print(f"[SDG]     - Table: '{table_asset.name}' ({time.time() - start_time:.2f}s)")
 92    await omni.kit.app.get_app().next_update_async()
 93
 94    # Keep only colliders on the table (disable rigid body dynamics)
 95    UsdPhysics.RigidBodyAPI(table_prim).GetRigidBodyEnabledAttr().Set(False)
 96
 97    # Compute table dimensions from its bounding box
 98    bbox_cache = UsdGeom.BBoxCache(Usd.TimeCode.Default(), includedPurposes=[UsdGeom.Tokens.default_])
 99    table_bbox = bbox_cache.ComputeWorldBound(table_prim)
100    table_extent = table_bbox.GetRange().GetSize()
101
102    # Choose a random dish and add it to the stage
103    dish_asset = dishes[rng.integers(len(dishes))]
104    start_time = time.time()
105    dish_prim = rep.functional.create.reference(usd_path=dish_asset.main_url, parent="/Assets", name=dish_asset.name)
106    set_prim_variants(dish_prim, variants)
107    upgrade_prim_semantics_to_labels(dish_prim)
108    print(f"[SDG]     - Dish: '{dish_asset.name}' ({time.time() - start_time:.2f}s)")
109    await omni.kit.app.get_app().next_update_async()
110
111    # Compute dish dimensions from its bounding box
112    dish_bbox = bbox_cache.ComputeWorldBound(dish_prim)
113    dish_extent = dish_bbox.GetRange().GetSize()
114
115    # Calculate random position for the dish near the center of the table
116    center_region_scale = 0.75
117    dish_range_x = max(0, (table_extent[0] - dish_extent[0]) / 2 * center_region_scale)
118    dish_range_y = max(0, (table_extent[1] - dish_extent[1]) / 2 * center_region_scale)
119    dish_position = (
120        rng.uniform(-dish_range_x, dish_range_x) if dish_range_x > 0 else 0,
121        rng.uniform(-dish_range_y, dish_range_y) if dish_range_y > 0 else 0,
122        table_extent[2] + dish_extent[2] / 2,
123    )
124    dish_prim.GetAttribute("xformOp:translate").Set(dish_position)
125
126    # Add random items above the dish
127    num_items = rng.integers(2, 5)
128    item_prims = []
129    for _ in range(num_items):
130        item_asset = items[rng.integers(len(items))]
131        start_time = time.time()
132        item_prim = rep.functional.create.reference(
133            usd_path=item_asset.main_url, parent="/Assets", name=item_asset.name
134        )
135        set_prim_variants(item_prim, variants)
136        upgrade_prim_semantics_to_labels(item_prim)
137        print(f"[SDG]     - Item: '{item_asset.name}' ({time.time() - start_time:.2f}s)")
138        item_prims.append(item_prim)
139        await omni.kit.app.get_app().next_update_async()
140
141    # Position items stacked above the dish
142    print(f"[SDG]   Positioning assets on table...")
143    stack_height = dish_position[2]
144    item_scatter_radius = max(0, dish_extent[0] / 4)
145    for item_prim in item_prims:
146        item_bbox = bbox_cache.ComputeWorldBound(item_prim)
147        item_extent = item_bbox.GetRange().GetSize()
148        scatter_x = rng.uniform(-item_scatter_radius, item_scatter_radius) if item_scatter_radius > 0 else 0
149        scatter_y = rng.uniform(-item_scatter_radius, item_scatter_radius) if item_scatter_radius > 0 else 0
150        item_position = (
151            dish_position[0] + scatter_x,
152            dish_position[1] + scatter_y,
153            stack_height + item_extent[2] / 2,
154        )
155        item_prim.GetAttribute("xformOp:translate").Set(item_position)
156        stack_height += item_extent[2]
157
158    # Run physics simulation for items to settle
159    num_sim_steps = 25
160    print(f"[SDG]   Running physics simulation ({num_sim_steps} steps)...")
161    timeline = omni.timeline.get_timeline_interface()
162    timeline.play()
163    for _ in range(num_sim_steps):
164        await omni.kit.app.get_app().next_update_async()
165    timeline.pause()
166
167    print(f"[SDG]   Setting edit target to root layer...")
168    stage.SetEditTarget(root_layer)
169
170    print(f"[SDG]   Positioning camera and capturing frame...")
171    camera_position = (
172        dish_position[0] + rng.uniform(-0.5, 0.5),
173        dish_position[1] + rng.uniform(-0.5, 0.5),
174        dish_position[2] + 1.5 + rng.uniform(-0.5, 0.5),
175    )
176    rep.functional.modify.pose(
177        camera_prim, position_value=camera_position, look_at_value=dish_prim, look_at_up_axis=(0, 0, 1)
178    )
179    render_product.hydra_texture.set_updates_enabled(True)
180    await rep.orchestrator.step_async(delta_time=0.0, rt_subframes=16)
181    render_product.hydra_texture.set_updates_enabled(False)
182
183    print(f"[SDG]   Removing temp variation layer...")
184    variation_layer.Clear()
185    root_layer.subLayerPaths.remove(variation_layer.identifier)
186
187
188async def run_simready_randomizations_async(num_scenarios: int) -> None:
189    """Run multiple SimReady randomization scenarios and capture the results."""
190    print(f"[SDG] Initializing scene...")
191    await omni.usd.get_context().new_stage_async()
192    stage = omni.usd.get_context().get_stage()
193
194    # Initialize randomization
195    rng = np.random.default_rng(34)
196    rep.set_global_seed(34)
197
198    # Data capture will happen manually using step()
199    rep.orchestrator.set_capture_on_play(False)
200
201    # Set DLSS to Quality mode (2) for best SDG results , options: 0 (Performance), 1 (Balanced), 2 (Quality), 3 (Auto)
202    carb.settings.get_settings().set("rtx/post/dlss/execMode", 2)
203
204    # Add lights to the scene
205    print(f"[SDG] Setting up lighting...")
206    rep.functional.create.xform(name="World")
207    rep.functional.create.dome_light(intensity=500, parent="/World", name="DomeLight")
208    rep.functional.create.distant_light(intensity=2500, parent="/World", name="DistantLight", rotation=(-75, 0, 0))
209
210    # Simready explorer window needs to be created for the search to work
211    enable_simready_explorer()
212
213    # Search for the simready assets
214    tables, dishes, items = await search_assets_async()
215
216    # Create the writer and the render product for capturing the scene
217    output_dir = os.path.join(os.getcwd(), "_out_simready_assets")
218    backend = rep.backends.get("DiskBackend")
219    backend.initialize(output_dir=output_dir)
220    writer = rep.writers.get("BasicWriter")
221    print(f"[SDG] Initializing writer, output directory: {output_dir}...")
222    writer.initialize(backend=backend, rgb=True)
223
224    # Create camera and render product (disabled by default, enabled only when capturing)
225    print(f"[SDG] Creating camera and render product...")
226    camera_prim = rep.functional.create.camera(position=(5, 5, 5), look_at=(0, 0, 0), parent="/World", name="Camera")
227    rp = rep.create.render_product(camera_prim, (512, 512))
228    rp.hydra_texture.set_updates_enabled(False)
229    writer.attach(rp)
230
231    # Generate randomized scenarios
232    for i in range(num_scenarios):
233        print(f"[SDG] Scenario {i + 1}/{num_scenarios}")
234        await run_simready_randomization_async(
235            stage=stage, camera_prim=camera_prim, render_product=rp, tables=tables, dishes=dishes, items=items, rng=rng
236        )
237
238    # Finalize and cleanup
239    print("[SDG] Wait for the data to be written and cleanup render products...")
240    await rep.orchestrator.wait_until_complete_async()
241    writer.detach()
242    rp.destroy()
243
244
245num_scenarios = 5
246print(f"[SDG] Starting SDG pipeline with {num_scenarios} scenarios...")
247asyncio.ensure_future(run_simready_randomizations_async(num_scenarios))
Simready Assets SDG Example
  1from isaacsim import SimulationApp
  2
  3simulation_app = SimulationApp(launch_config={"headless": False})
  4
  5import argparse
  6import asyncio
  7import os
  8import time
  9
 10import carb.settings
 11import numpy as np
 12import omni.kit.app
 13import omni.replicator.core as rep
 14import omni.timeline
 15import omni.usd
 16from isaacsim.core.utils.semantics import upgrade_prim_semantics_to_labels
 17from pxr import Gf, Sdf, Usd, UsdGeom, UsdPhysics
 18
 19parser = argparse.ArgumentParser()
 20parser.add_argument("--num_scenarios", type=int, default=5, help="Number of randomization scenarios to create")
 21args, _ = parser.parse_known_args()
 22num_scenarios = args.num_scenarios
 23
 24# Make sure the simready explorer extension is enabled
 25ext_manager = omni.kit.app.get_app().get_extension_manager()
 26if not ext_manager.is_extension_enabled("omni.simready.explorer"):
 27    ext_manager.set_extension_enabled_immediate("omni.simready.explorer", True)
 28import omni.simready.explorer as sre
 29
 30
 31def enable_simready_explorer() -> None:
 32    """Enable the SimReady Explorer window if not already open."""
 33    if sre.get_instance().browser_model is None:
 34        import omni.kit.actions.core as actions
 35
 36        actions.execute_action("omni.simready.explorer", "toggle_window")
 37
 38
 39def set_prim_variants(prim: Usd.Prim, variants: dict[str, str]) -> None:
 40    """Set variant selections on a prim from a dictionary of variant set names to values."""
 41    vsets = prim.GetVariantSets()
 42    for name, value in variants.items():
 43        vset = vsets.GetVariantSet(name)
 44        if vset:
 45            vset.SetVariantSelection(value)
 46
 47
 48async def search_assets_async() -> tuple[list, list, list]:
 49    """Search for SimReady assets (tables, dishes, items) asynchronously."""
 50    print(f"[SDG] Searching for SimReady assets...")
 51    start_time = time.time()
 52    tables = await sre.find_assets(["table", "furniture"])
 53    print(f"[SDG]   - Found {len(tables)} tables ({time.time() - start_time:.2f}s)")
 54    start_time = time.time()
 55    plates = await sre.find_assets(["plate"])
 56    print(f"[SDG]   - Found {len(plates)} plates ({time.time() - start_time:.2f}s)")
 57    start_time = time.time()
 58    bowls = await sre.find_assets(["bowl"])
 59    print(f"[SDG]   - Found {len(bowls)} bowls ({time.time() - start_time:.2f}s)")
 60    dishes = plates + bowls
 61    start_time = time.time()
 62    fruits = await sre.find_assets(["fruit"])
 63    print(f"[SDG]   - Found {len(fruits)} fruits ({time.time() - start_time:.2f}s)")
 64    start_time = time.time()
 65    vegetables = await sre.find_assets(["vegetable"])
 66    print(f"[SDG]   - Found {len(vegetables)} vegetables ({time.time() - start_time:.2f}s)")
 67    items = fruits + vegetables
 68    return tables, dishes, items
 69
 70
 71def run_simready_randomization(
 72    stage: Usd.Stage,
 73    camera_prim: Usd.Prim,
 74    render_product,
 75    tables: list,
 76    dishes: list,
 77    items: list,
 78    rng: np.random.Generator = None,
 79) -> None:
 80    """Randomize a scene with SimReady assets, run physics, and capture the result."""
 81    if rng is None:
 82        rng = np.random.default_rng()
 83
 84    print(f"[SDG]   Creating anonymous variation layer for the randomizations...")
 85    root_layer = stage.GetRootLayer()
 86    variation_layer = Sdf.Layer.CreateAnonymous("variation")
 87    root_layer.subLayerPaths.insert(0, variation_layer.identifier)
 88    stage.SetEditTarget(variation_layer)
 89
 90    # Load the simready assets with rigid body properties
 91    variants = {"PhysicsVariant": "RigidBody"}
 92    rep.functional.create.scope(name="Assets")
 93
 94    # Choose a random table and add it to the stage
 95    print(f"[SDG]   Loading assets...")
 96    table_asset = tables[rng.integers(len(tables))]
 97    start_time = time.time()
 98    table_prim = rep.functional.create.reference(usd_path=table_asset.main_url, parent="/Assets", name=table_asset.name)
 99    set_prim_variants(table_prim, variants)
100    upgrade_prim_semantics_to_labels(table_prim)
101    print(f"[SDG]     - Table: '{table_asset.name}' ({time.time() - start_time:.2f}s)")
102    simulation_app.update()
103
104    # Keep only colliders on the table (disable rigid body dynamics)
105    UsdPhysics.RigidBodyAPI(table_prim).GetRigidBodyEnabledAttr().Set(False)
106
107    # Compute table dimensions from its bounding box
108    bbox_cache = UsdGeom.BBoxCache(Usd.TimeCode.Default(), includedPurposes=[UsdGeom.Tokens.default_])
109    table_bbox = bbox_cache.ComputeWorldBound(table_prim)
110    table_extent = table_bbox.GetRange().GetSize()
111
112    # Choose a random dish and add it to the stage
113    dish_asset = dishes[rng.integers(len(dishes))]
114    start_time = time.time()
115    dish_prim = rep.functional.create.reference(usd_path=dish_asset.main_url, parent="/Assets", name=dish_asset.name)
116    set_prim_variants(dish_prim, variants)
117    upgrade_prim_semantics_to_labels(dish_prim)
118    print(f"[SDG]     - Dish: '{dish_asset.name}' ({time.time() - start_time:.2f}s)")
119    simulation_app.update()
120
121    # Compute dish dimensions from its bounding box
122    dish_bbox = bbox_cache.ComputeWorldBound(dish_prim)
123    dish_extent = dish_bbox.GetRange().GetSize()
124
125    # Calculate random position for the dish near the center of the table
126    center_region_scale = 0.75
127    dish_range_x = max(0, (table_extent[0] - dish_extent[0]) / 2 * center_region_scale)
128    dish_range_y = max(0, (table_extent[1] - dish_extent[1]) / 2 * center_region_scale)
129    dish_position = (
130        rng.uniform(-dish_range_x, dish_range_x) if dish_range_x > 0 else 0,
131        rng.uniform(-dish_range_y, dish_range_y) if dish_range_y > 0 else 0,
132        table_extent[2] + dish_extent[2] / 2,
133    )
134    dish_prim.GetAttribute("xformOp:translate").Set(dish_position)
135
136    # Add random items above the dish
137    num_items = rng.integers(2, 5)
138    item_prims = []
139    for _ in range(num_items):
140        item_asset = items[rng.integers(len(items))]
141        start_time = time.time()
142        item_prim = rep.functional.create.reference(
143            usd_path=item_asset.main_url, parent="/Assets", name=item_asset.name
144        )
145        set_prim_variants(item_prim, variants)
146        upgrade_prim_semantics_to_labels(item_prim)
147        print(f"[SDG]     - Item: '{item_asset.name}' ({time.time() - start_time:.2f}s)")
148        item_prims.append(item_prim)
149        simulation_app.update()
150
151    # Position items stacked above the dish
152    print(f"[SDG]   Positioning assets on table...")
153    stack_height = dish_position[2]
154    item_scatter_radius = max(0, dish_extent[0] / 4)
155    for item_prim in item_prims:
156        item_bbox = bbox_cache.ComputeWorldBound(item_prim)
157        item_extent = item_bbox.GetRange().GetSize()
158        scatter_x = rng.uniform(-item_scatter_radius, item_scatter_radius) if item_scatter_radius > 0 else 0
159        scatter_y = rng.uniform(-item_scatter_radius, item_scatter_radius) if item_scatter_radius > 0 else 0
160        item_position = (
161            dish_position[0] + scatter_x,
162            dish_position[1] + scatter_y,
163            stack_height + item_extent[2] / 2,
164        )
165        item_prim.GetAttribute("xformOp:translate").Set(item_position)
166        stack_height += item_extent[2]
167
168    # Run physics simulation for items to settle
169    num_sim_steps = 25
170    print(f"[SDG]   Running physics simulation ({num_sim_steps} steps)...")
171    timeline = omni.timeline.get_timeline_interface()
172    timeline.play()
173    for _ in range(num_sim_steps):
174        simulation_app.update()
175    timeline.pause()
176
177    print(f"[SDG]   Setting edit target to root layer...")
178    stage.SetEditTarget(root_layer)
179
180    print(f"[SDG]   Positioning camera and capturing frame...")
181    camera_position = (
182        dish_position[0] + rng.uniform(-0.5, 0.5),
183        dish_position[1] + rng.uniform(-0.5, 0.5),
184        dish_position[2] + 1.5 + rng.uniform(-0.5, 0.5),
185    )
186    rep.functional.modify.pose(
187        camera_prim, position_value=camera_position, look_at_value=dish_prim, look_at_up_axis=(0, 0, 1)
188    )
189    render_product.hydra_texture.set_updates_enabled(True)
190    rep.orchestrator.step(delta_time=0.0, rt_subframes=16)
191    render_product.hydra_texture.set_updates_enabled(False)
192
193    print(f"[SDG]   Removing temp variation layer...")
194    variation_layer.Clear()
195    root_layer.subLayerPaths.remove(variation_layer.identifier)
196
197
198def run_simready_randomizations(num_scenarios: int) -> None:
199    """Run multiple SimReady randomization scenarios and capture the results."""
200    print(f"[SDG] Initializing scene...")
201    omni.usd.get_context().new_stage()
202    stage = omni.usd.get_context().get_stage()
203
204    # Initialize randomization
205    rng = np.random.default_rng(34)
206    rep.set_global_seed(34)
207
208    # Data capture will happen manually using step()
209    rep.orchestrator.set_capture_on_play(False)
210
211    # Set DLSS to Quality mode (2) for best SDG results , options: 0 (Performance), 1 (Balanced), 2 (Quality), 3 (Auto)
212    carb.settings.get_settings().set("rtx/post/dlss/execMode", 2)
213
214    # Add lights to the scene
215    print(f"[SDG] Setting up lighting...")
216    rep.functional.create.xform(name="World")
217    rep.functional.create.dome_light(intensity=500, parent="/World", name="DomeLight")
218    rep.functional.create.distant_light(intensity=2500, parent="/World", name="DistantLight", rotation=(-75, 0, 0))
219
220    # Simready explorer window needs to be created for the search to work
221    enable_simready_explorer()
222
223    # Search for the simready assets and wait until the task is complete
224    search_task = asyncio.ensure_future(search_assets_async())
225    while not search_task.done():
226        simulation_app.update()
227    tables, dishes, items = search_task.result()
228
229    # Create the writer and the render product for capturing the scene
230    output_dir = os.path.join(os.getcwd(), "_out_simready_assets")
231    backend = rep.backends.get("DiskBackend")
232    backend.initialize(output_dir=output_dir)
233    writer = rep.writers.get("BasicWriter")
234    print(f"[SDG] Initializing writer, output directory: {output_dir}...")
235    writer.initialize(backend=backend, rgb=True)
236
237    # Create camera and render product (disabled by default, enabled only when capturing)
238    print(f"[SDG] Creating camera and render product...")
239    camera_prim = rep.functional.create.camera(position=(5, 5, 5), look_at=(0, 0, 0), parent="/World", name="Camera")
240    rp = rep.create.render_product(camera_prim, (512, 512))
241    rp.hydra_texture.set_updates_enabled(False)
242    writer.attach(rp)
243
244    # Generate randomized scenarios
245    for i in range(num_scenarios):
246        print(f"[SDG] Scenario {i + 1}/{num_scenarios}")
247        run_simready_randomization(
248            stage=stage, camera_prim=camera_prim, render_product=rp, tables=tables, dishes=dishes, items=items, rng=rng
249        )
250
251    # Finalize and cleanup
252    print("[SDG] Wait for the data to be written and cleanup render products...")
253    rep.orchestrator.wait_until_complete()
254    writer.detach()
255    rp.destroy()
256
257
258print(f"[SDG] Starting SDG pipeline with {num_scenarios} scenarios...")
259run_simready_randomizations(num_scenarios)
260
261simulation_app.close()