docs/3D_ASSET_PIPELINE.md

@@ -20,7 +20,8 @@ menu dropdown, fully animated in first AND third person, synced in multiplayer.
 3. **Animation library** — already bundled. `assets/characters/animations/_library.glb`
    is the CC0 Quaternius Universal Animation Library (13 game-relevant clips
    mapped in `LIBRARY_CLIP_MAP`). `tools/autorig.py` rigs every character to
-   this library's own skeleton, so clips apply with zero retargeting. To use a
+   this library's skeleton (fitting the arm bones to the model's actual pose),
+   and `merge_animations.py` retargets each clip onto that fitted rest. To use a
    different/larger set instead, see "Swapping the animation library" below.
 
 ## The pipeline, step by step
@@ -31,8 +32,8 @@ Each step is also runnable on its own:
 |---|---|---|
 | 1. Find | `python tools/sketchfab_import.py search "anime robot" --rigged` | Search downloadable models (license shown per result) |
 | 2. Download | `python tools/sketchfab_import.py download <uid>` | GLB + license/attribution JSON into `assets/characters/incoming/` |
-| 3. Auto-rig | `blender --background --python tools/autorig.py -- in.glb out.glb` | Fits a Mixamo-named skeleton, binds automatic weights |
-| 4. Animate | `blender --background --python tools/merge_animations.py -- rigged.glb assets/characters/animations out.glb` | Merges the shared clip library, strips root motion, canonical names |
+| 3. Auto-rig | `blender --background --python tools/autorig.py -- in.glb out.glb` | Fits the skeleton (incl. arm bones) to the mesh, binds smooth nearest-bone weights |
+| 4. Animate | `blender --background --python tools/merge_animations.py -- rigged.glb assets/characters/animations out.glb` | Retargets the clip library onto the fitted rest, strips root motion, canonical names |
 | 5. Register | (automatic in pipeline.py) | Copies to `skins/`, adds entry to `skins.json` |
 
 `tools/pipeline.py` chains all of it. Useful flags:

tools/autorig.py

@@ -127,11 +127,58 @@ def mesh_slice_width(obj, y_frac, height, min_z):
     return max_x
 
 
+def fit_arms_to_mesh(arm, mesh):
+    """Rotate each arm bone chain to lie along the mesh's actual arm direction.
+
+    Sketchfab humanoids are usually modelled in an A-pose (arms angled down),
+    but the library skeleton rests in a T-pose (arms horizontal). If we bind the
+    A-pose mesh to T-pose arm bones, bone-heat weighting fails (bones sit outside
+    the arms) and the animations shove the already-drooped arms across the body.
+
+    Reorienting the arm bones down the real arm axis makes the rest pose fit the
+    mesh, so bone-heat succeeds. The animations are then retargeted from the
+    library's T-pose rest to this fitted rest in merge_animations.py.
+    """
+    mw = mesh.matrix_world
+    verts = [mw @ v.co for v in mesh.data.vertices]
+    bpy.context.view_layer.objects.active = arm
+    bpy.ops.object.mode_set(mode="EDIT")
+    eb = arm.data.edit_bones
+    for side, sx in (("L", 1.0), ("R", -1.0)):
+        chain = [eb.get(f"DEF-upper_arm.{side}"), eb.get(f"DEF-forearm.{side}"),
+                 eb.get(f"DEF-hand.{side}")]
+        chain = [b for b in chain if b]
+        if not chain:
+            continue
+        shoulder = chain[0].head.copy()
+        # Hand tip = farthest mesh vertex to this side, near arm height.
+        arm_z = shoulder.z
+        cand = [v for v in verts if (v.x * sx) > abs(shoulder.x) * 0.8
+                and abs(v.z - arm_z) < 0.35]
+        if not cand:
+            continue
+        hand_tip = max(cand, key=lambda v: v.x * sx)
+        cur = chain[-1].tail - shoulder
+        tgt = hand_tip - shoulder
+        if cur.length < 1e-4 or tgt.length < 1e-4:
+            continue
+        rot = cur.normalized().rotation_difference(tgt.normalized())
+        # Rotate the whole chain about the shoulder joint.
+        for b in chain:
+            b.head = shoulder + rot @ (b.head - shoulder)
+            b.tail = shoulder + rot @ (b.tail - shoulder)
+        sh = eb.get(f"DEF-shoulder.{side}")
+        if sh:
+            sh.tail = chain[0].head
+    bpy.ops.object.mode_set(mode="OBJECT")
+    print("Fitted arm bones to mesh A-pose")
+
+
 def import_library_armature(height):
     """Import the animation library's armature as the rig template.
 
-    Rigging to the exact skeleton the clips were authored on means zero
-    retargeting: merge_animations.py can apply every clip verbatim.
+    Rigging to the exact skeleton the clips were authored on means clips need
+    only a rest-pose retarget (handled in merge_animations.py).
     """
     before = set(bpy.data.objects)
     before_actions = set(bpy.data.actions)
@@ -230,26 +277,42 @@ def _count_weighted_verts(mesh):
     return sum(1 for v in mesh.data.vertices if len(v.groups) > 0)
 
 
-def rigid_nearest_bone_weights(mesh, arm):
-    """Assign every vertex fully to its nearest deform bone.
+# Bones that must never deform the mesh: the root/master and any rig control
+# or mechanism bones. A deform-flagged root runs up the body centre-line, so
+# without this it captures inner-leg/heel verts and stretches them to centre.
+_NON_DEFORM_HINTS = ("root", "master", "mch-", "mch_", "ctrl", "org-", "-ik",
+                     "_ik", "pole", "target", "properties")
 
-    A guaranteed-to-work fallback when bone-heat weighting fails (common on
-    imported meshes with layered hair/clothing). Produces rigid skinning — no
-    smooth falloff at joints — but the character follows the skeleton and every
-    animation plays. Good enough for a fast-paced shooter at gameplay distance.
-    """
-    # Build (name, head, tail) for deform bones in world space.
+
+def _deform_segments(arm):
     segs = []
     for b in arm.data.bones:
         if not b.use_deform:
             continue
+        if any(h in b.name.lower() for h in _NON_DEFORM_HINTS):
+            continue
         head = arm.matrix_world @ b.head_local
         tail = arm.matrix_world @ b.tail_local
         segs.append((b.name, head, tail))
+    return segs
+
+
+def rigid_nearest_bone_weights(mesh, arm):
+    """Weight each vertex to its nearest deform bones (distance-falloff blend).
+
+    A guaranteed-to-work substitute when bone-heat weighting fails — which it
+    does on most imported characters, whose joined hair/clothing/body meshes
+    have the interior and overlapping geometry the heat solver chokes on.
+
+    Each vertex is blended across its nearest few bone segments with an
+    inverse-distance falloff, so joints deform smoothly instead of tearing.
+    Control/root bones are excluded (see _NON_DEFORM_HINTS) so the mesh follows
+    real limbs, not the centre-line master bone.
+    """
+    segs = _deform_segments(arm)
     if not segs:
         return 0
 
-    # Fresh vertex groups.
     for vg in list(mesh.vertex_groups):
         mesh.vertex_groups.remove(vg)
     groups = {name: mesh.vertex_groups.new(name=name) for name, _, _ in segs}
@@ -260,11 +323,19 @@ def rigid_nearest_bone_weights(mesh, arm):
         t = 0.0 if denom == 0 else max(0.0, min(1.0, (p - a).dot(ab) / denom))
         return (p - (a + ab * t)).length
 
+    K = 4          # blend across up to this many nearest bones
+    FALLOFF = 3.0  # higher = tighter to the single nearest bone
     mw = mesh.matrix_world
     for v in mesh.data.vertices:
         p = mw @ v.co
-        best_name = min(segs, key=lambda s: dist_to_seg(p, s[1], s[2]))[0]
-        groups[best_name].add([v.index], 1.0, "REPLACE")
+        dists = sorted(((dist_to_seg(p, s[1], s[2]), s[0]) for s in segs),
+                       key=lambda d: d[0])[:K]
+        nearest = max(dists[0][0], 1e-5)
+        # Inverse-distance weights, relative to the nearest bone.
+        raw = [((nearest / max(d, 1e-5)) ** FALLOFF, name) for d, name in dists]
+        total = sum(w for w, _ in raw)
+        for w, name in raw:
+            groups[name].add([v.index], w / total, "REPLACE")
     return _count_weighted_verts(mesh)
 
 
@@ -357,15 +428,24 @@ def main():
         height = TARGET_HEIGHT
 
         arm = None
+        from_library = False
         if LIBRARY:
             print(f"Using animation library skeleton: {LIBRARY}")
             arm = import_library_armature(height)
+            from_library = arm is not None
         if not arm:
             shoulder_w = mesh_slice_width(mesh, 0.80, height, min_z)
             hip_w = mesh_slice_width(mesh, 0.53, height, min_z)
             print(f"Fitted: height={height:.2f} shoulder_w={shoulder_w:.2f} hip_w={hip_w:.2f}")
             arm = build_mixamo_armature(height, min_z, shoulder_w, hip_w)
 
+        # Fit the arm bones to the mesh's real (usually A-pose) arm direction so
+        # bone-heat weighting fits and the rest pose matches the geometry. The
+        # library-clip retarget in merge_animations.py compensates for the
+        # rest-pose change.
+        if from_library:
+            fit_arms_to_mesh(arm, mesh)
+
         bind_mesh_to_armature(mesh, arm)
 
     print(f"Exporting {OUTPUT}...")

tools/merge_animations.py

@@ -230,8 +230,68 @@ def _add_clip_track(target_arm, action, clip_name):
     action.use_fake_user = True
 
 
+def _assign_action(obj, action):
+    """Assign an action to an object across Blender versions (slotted actions)."""
+    if not obj.animation_data:
+        obj.animation_data_create()
+    obj.animation_data.action = action
+    try:
+        slots = action.slots
+        if len(slots):
+            obj.animation_data.action_slot = slots[0]
+    except Exception:
+        pass
+
+
+def retarget_action(target_arm, src_arm, src_action, clip_name):
+    """Retarget one source clip onto the character rig via world-space
+    constraint baking.
+
+    The character rig's arms were fitted to the mesh's A-pose (autorig.py), so
+    its rest differs from the library's T-pose. Copying each bone's WORLD
+    orientation (not its local channel) reproduces the library's actual motion
+    regardless of that rest difference, then baking captures it as a clean
+    action on the character rig.
+    """
+    _assign_action(src_arm, src_action)
+    fr0 = int(src_action.frame_range[0])
+    fr1 = int(src_action.frame_range[1])
+
+    for pb in target_arm.pose.bones:
+        if pb.name not in src_arm.pose.bones:
+            continue
+        cr = pb.constraints.new("COPY_ROTATION")
+        cr.target = src_arm
+        cr.subtarget = pb.name
+        cr.target_space = "WORLD"
+        cr.owner_space = "WORLD"
+        if pb.name.lower().endswith("hips"):
+            cl = pb.constraints.new("COPY_LOCATION")
+            cl.target = src_arm
+            cl.subtarget = pb.name
+            cl.target_space = "WORLD"
+            cl.owner_space = "WORLD"
+
+    bpy.ops.object.select_all(action="DESELECT")
+    target_arm.select_set(True)
+    bpy.context.view_layer.objects.active = target_arm
+    if target_arm.animation_data:
+        target_arm.animation_data.action = None
+
+    bpy.ops.nla.bake(frame_start=fr0, frame_end=fr1, step=1,
+                     only_selected=False, visual_keying=True,
+                     clear_constraints=True, clear_parents=False,
+                     use_current_action=True, bake_types={"POSE"})
+
+    baked = target_arm.animation_data.action
+    if STRIP_ROOT_MOTION:
+        strip_hips_horizontal(baked)
+    return baked
+
+
 def merge_from_library(target_arm, target_hips, library_path):
-    """Merge clips from a single multi-clip library GLB (same skeleton)."""
+    """Retarget every mapped clip from a single multi-clip library GLB onto the
+    character rig (whose arm rest was fitted to the mesh)."""
     print(f"Using multi-clip library: {library_path}")
     before = set(bpy.data.objects)
     before_actions = set(bpy.data.actions)
@@ -239,31 +299,39 @@ def merge_from_library(target_arm, target_hips, library_path):
     new_objects = [o for o in bpy.data.objects if o not in before]
     new_actions = [a for a in bpy.data.actions if a not in before_actions]
     src_arm = find_armature(new_objects)
+    if not src_arm:
+        print("ERROR: no armature in animation library")
+        sys.exit(1)
 
-    ratio = 1.0
-    if src_arm:
-        ratio = target_hips / max(hips_height(src_arm), 1e-6)
-    print(f"Location scale ratio: {ratio:.3f}")
+    # Match the library rig's height to the character so hips bob / foot planting
+    # translate correctly under the world-space location copy.
+    src_hips = hips_height(src_arm)
+    if src_hips > 1e-4:
+        s = target_hips / src_hips
+        src_arm.scale = (s, s, s)
+        bpy.ops.object.select_all(action="DESELECT")
+        src_arm.select_set(True)
+        bpy.context.view_layer.objects.active = src_arm
+        bpy.ops.object.transform_apply(location=False, rotation=False, scale=True)
 
     merged = 0
-    keep: list = []
+    keep = []
     for action in new_actions:
         base = action.name.split(".")[0]  # tolerate .001 suffixes
         clip_name = LIBRARY_CLIP_MAP.get(base)
         if not clip_name:
             continue
-        print(f"--- {base} -> '{clip_name}'")
-        scale_location_curves(action, ratio)
-        if STRIP_ROOT_MOTION:
-            strip_hips_horizontal(action)
-        _add_clip_track(target_arm, action, clip_name)
-        keep.append(action)
+        print(f"--- {base} -> '{clip_name}' (retarget bake)")
+        baked = retarget_action(target_arm, src_arm, action, clip_name)
+        target_arm.animation_data.action = None
+        _add_clip_track(target_arm, baked, clip_name)
+        keep.append(baked)
         merged += 1
 
     for o in new_objects:
         bpy.data.objects.remove(o, do_unlink=True)
-    for a in new_actions:
-        if a not in keep:
+    for a in list(bpy.data.actions):
+        if a not in keep and a in new_actions:
             bpy.data.actions.remove(a)
     return merged