Vastly improve Merge Doubles

- removed advanced merge doubles, it just does advanced by default
- same behavior as advanced was before, but now completes the task in under a second. Thanks to the power of BMesh!
- Labels now reflect this change

functions/optimization/remove_doubles.py

@@ -1,3 +1,4 @@
+import traceback
 import bpy
 import numpy as np
 from typing import List, TypedDict, Any, Literal, TypeAlias, cast
@@ -9,6 +10,8 @@ from ...core.common import (
     get_all_meshes,
 )
 from ...core.armature_validation import validate_armature
+import bmesh
+import mathutils
 
 # Constants
 MERGE_ITERATION_COUNT = 20
@@ -19,83 +22,38 @@ ModalReturnType: TypeAlias = Literal['RUNNING_MODAL', 'FINISHED', 'CANCELLED']
 
 class MeshEntry(TypedDict):
     mesh: Object
-    shapekeys: list[str]
+    shapekeys: list[bpy.types.Object]
-    vertices: int
-    cur_vertex_pass: int
 
-def create_duplicate_for_merge(context: Context, mesh: Object, shapekey_name: str) -> Object:
+def create_duplicate_for_merge(context: Context, mesh: Object, shapekey_name: str = "") -> Object:
     """Creates a duplicate mesh object for merge testing"""
-    context.view_layer.objects.active = mesh
+    
+    if(shapekey_name != ""):
+        mesh.active_shape_key_index = mesh.data.shape_keys.key_blocks.find(shapekey_name)
+    
     bpy.ops.object.mode_set(mode='OBJECT')
     bpy.ops.object.select_all(action='DESELECT')
     mesh.select_set(True)
+    context.view_layer.objects.active = mesh
     bpy.ops.object.duplicate()
+    if(shapekey_name != ""):
         bpy.ops.object.shape_key_move(type='TOP')
+        bpy.ops.object.shape_key_remove(all=True,apply_mix=False)
     
     duplicate = context.view_layer.objects.active
+    if(shapekey_name != ""):
         duplicate.name = f"{shapekey_name}_object_is_{mesh.name}"
+    
+    else:
+        duplicate.name = f"object_is_{mesh.name}"
     return duplicate
 
-def process_vertex_merging(mesh_data: bpy.types.Mesh, vertices_original: dict[int, Any], current_vertex: int) -> list[int]:
+def select_obj(context: Context, obj: Object, target_mode='OBJECT'):
-    """Process vertex merging and return merged vertex indices"""
-    merged_vertices = []
-    i, j = 0, 0
-    
-    while i < len(vertices_original):
-        if j + 1 > len(mesh_data.vertices):
-            merged_vertices.append(i)
-            j = j - 1
-        elif mesh_data.vertices[j].co.xyz != vertices_original[i]:
-            merged_vertices.append(i)
-            j = j - 1
-        elif vertices_original[i] == vertices_original[current_vertex]:
-            merged_vertices.append(i)
-        i, j = i + 1, j + 1
-    
-    return merged_vertices
-
-def vertex_moves(mesh_data: bpy.types.Mesh, vertex: int) -> bool:
-
-    for shapekey in mesh_data.shape_keys.key_blocks:
-        data: bpy.types.ShapeKey = shapekey
-
-        if data.points[vertex].co.xyz != mesh_data.vertices[vertex].co.xyz:
-            return True
-
-    return False
-
-def merge_vertex_at_index(mesh_data: bpy.types.Mesh, index: int, distance: float):
-
-    select_target_vertex = [False]*len(mesh_data.vertices)
-    select_target_vertex[index] = True
-
-    bpy.ops.object.mode_set(mode='OBJECT')
-    mesh_data.vertices.foreach_set("select",select_target_vertex)
-    bpy.ops.object.mode_set(mode='EDIT')
-    for _ in range(0,20): #for some reason, if using merge to unselected on a vertex, the vertex will only merge to 1 other vertex. so we gotta spam it to fix it.
-        bpy.ops.mesh.remove_doubles(threshold=distance, use_unselected=True, use_sharp_edge_from_normals=False)
     bpy.ops.object.mode_set(mode='OBJECT')
+    bpy.ops.object.select_all(action='DESELECT')
+    obj.select_set(True)
+    context.view_layer.objects.active = obj
+    bpy.ops.object.mode_set(mode=target_mode)
 
-class AvatarToolkit_OT_RemoveDoublesAdvanced(Operator):
-    bl_idname = "avatar_toolkit.remove_doubles_advanced"
-    bl_label = t("Optimization.remove_doubles_advanced")
-    bl_description = t("Optimization.remove_doubles_advanced_desc")
-    bl_options = {'REGISTER', 'UNDO'}
-
-    @classmethod
-    def poll(cls, context: Context) -> bool:
-        """Check if the operator can be executed"""
-        armature = get_active_armature(context)
-        if not armature:
-            return False
-        valid, _, _ = validate_armature(armature)
-        return valid
-
-    def execute(self, context: Context) -> set[str]:
-        """Execute the advanced remove doubles operator"""
-        context.scene.avatar_toolkit.remove_doubles_advanced = True
-        bpy.ops.avatar_toolkit.remove_doubles('INVOKE_DEFAULT')
-        return {'RUNNING_MODAL'}
 
 class AvatarToolkit_OT_RemoveDoubles(Operator):
     bl_idname = "avatar_toolkit.remove_doubles"
@@ -104,7 +62,7 @@ class AvatarToolkit_OT_RemoveDoubles(Operator):
     bl_options = {'REGISTER', 'UNDO'}
 
     objects_to_do: list[MeshEntry] = []
-
+    merge_distance: bpy.props.FloatProperty(name=t("Optimization.merge_distance"), description=t("Optimization.merge_distance_desc"), default=.001)
     @classmethod
     def poll(cls, context: Context) -> bool:
         """Check if the operator can be executed"""
@@ -117,27 +75,27 @@ class AvatarToolkit_OT_RemoveDoubles(Operator):
     def draw(self, context: Context) -> None:
         """Draw the operator's UI"""
         layout = self.layout
-        layout.prop(context.scene.avatar_toolkit, "remove_doubles_merge_distance")
+        layout.prop(self, "merge_distance")
-        layout.label(text=t("Optimization.remove_doubles_warning"))
-        layout.label(text=t("Optimization.remove_doubles_wait"))
 
     def invoke(self, context: Context, event: Event) -> set[str]:
         """Initialize the operator"""
         logger.info("Starting modal execution of merge doubles safely")
         return context.window_manager.invoke_props_dialog(self)
 
-    def setup_mesh_entry(self, mesh: Object) -> MeshEntry:
+    def setup_mesh_entry(self, context: Context, mesh: Object) -> MeshEntry:
         """Set up mesh entry data structure"""
+        #create shapekey objects to merge doubles on.
+        shapes: list[bpy.types.Object] = []
+        if(mesh.data.shape_keys):
+            for shape in mesh.data.shape_keys.key_blocks:
+                shapes.append(create_duplicate_for_merge(context,mesh,shape.name))
+        else:
+            shapes.append(create_duplicate_for_merge(context,mesh))
         mesh_entry: MeshEntry = {
             "mesh": mesh,
-            "shapekeys": [],
+            "shapekeys": shapes
-            "vertices": len(mesh.data.vertices),
-            "cur_vertex_pass": 0
         }
 
-        if mesh.data.shape_keys:
-            mesh_entry["shapekeys"] = [shape.name for shape in mesh.data.shape_keys.key_blocks]
-        
         return mesh_entry
 
     def execute(self, context: Context) -> set[str]:
@@ -157,7 +115,7 @@ class AvatarToolkit_OT_RemoveDoubles(Operator):
             for mesh in objects:
                 if mesh.data.name not in [obj["mesh"].data.name for obj in self.objects_to_do]:
                     logger.debug(f"Setting up data for object {mesh.name}")
-                    mesh_entry = self.setup_mesh_entry(mesh)
+                    mesh_entry = self.setup_mesh_entry(context, mesh)
                     self.objects_to_do.append(mesh_entry)
 
             context.window_manager.modal_handler_add(self)
@@ -167,148 +125,50 @@ class AvatarToolkit_OT_RemoveDoubles(Operator):
             logger.error(f"Error in execute: {str(e)}")
             return {'CANCELLED'}
 
-    def modify_mesh(self, context: Context, mesh: MeshEntry) -> None:
-        """Basic mesh modification for simple cases"""
-        try:
-            mesh["mesh"].select_set(True)
-            context.view_layer.objects.active = mesh["mesh"]
-            mesh_data = mesh["mesh"].data
-            
-            bpy.ops.object.mode_set(mode='EDIT')
-            bpy.ops.object.mode_set(mode='OBJECT')
-            
-            # Select vertices with different positions in shape keys
-            for index, point in enumerate(mesh["mesh"].active_shape_key.points):
-                if point.co.xyz != mesh_data.shape_keys.key_blocks[0].points[index].co.xyz:
-                    mesh_data.vertices[index].select = True
-                    logger.debug(f"Shapekey has moved vertex at index {index}")
-                    
-            bpy.ops.object.mode_set(mode='EDIT')
-            bpy.ops.object.mode_set(mode='OBJECT')
-            mesh["mesh"].select_set(False)
-            
-        except Exception as e:
-            logger.error(f"Error in modify_mesh: {str(e)}")
-
-    def modify_mesh_advanced(self, context: Context, mesh_entry: MeshEntry) -> int:
-        """Advanced mesh modification with shape key handling"""
-        try:
-            final_merged_vertex_group = []
-            initialized_final = False
-            merge_distance = context.scene.avatar_toolkit.remove_doubles_merge_distance
-
-            for shapekey_name in mesh_entry["shapekeys"]:
-                duplicate = create_duplicate_for_merge(context, mesh_entry["mesh"], shapekey_name)
-                vertices_original = {i: v.co.xyz for i, v in enumerate(duplicate.data.vertices)}
-                
-                
-                merge_vertex_at_index(duplicate.data, mesh_entry["cur_vertex_pass"], merge_distance) #merge the vertex at our pass to find vertices that would merge to our vertex at this shapekey.
-
-                # Process merging
-                merged_vertices = process_vertex_merging(duplicate.data, vertices_original, mesh_entry["cur_vertex_pass"]) # find what vertices actually merged.
-                
-                if not initialized_final:
-                    final_merged_vertex_group = merged_vertices.copy()
-                    initialized_final = True
-                else:
-                    final_merged_vertex_group = [v for v in final_merged_vertex_group if v in merged_vertices] # remove vertices that merged from the list if they didn't merge during this shapkey.
-                bpy.ops.object.delete()
-
-            # Apply final merging
-            if final_merged_vertex_group:
-                self.apply_final_merging(context, mesh_entry, final_merged_vertex_group, merge_distance) # merge all vertices that merged on every shapekey no matter the shapekey during the loop.
-                
-            return len(final_merged_vertex_group)
-            
-        except Exception as e:
-            logger.error(f"Error in modify_mesh_advanced: {str(e)}")
-            return 1
-
-    def apply_final_merging(self, context: Context, mesh_entry: MeshEntry, vertex_group: list[int], merge_distance: float) -> None:
-        """Apply final vertex merging operations"""
-        mesh = mesh_entry["mesh"]
-        context.view_layer.objects.active = mesh
-        mesh.select_set(True)
-        
-        bpy.ops.object.mode_set(mode='OBJECT')
-        select_target_group = [False] * len(mesh.data.vertices)
-        for vertex_index in vertex_group:
-            select_target_group[vertex_index] = True
-            
-        mesh.data.vertices.foreach_set("select", select_target_group)
-        bpy.ops.object.mode_set(mode='EDIT')
-        bpy.ops.mesh.remove_doubles(threshold=merge_distance, use_unselected=False)
-        bpy.ops.object.mode_set(mode='OBJECT')
-
-    def process_simple_mesh(self, context: Context, mesh: MeshEntry, merge_distance: float) -> None:
-        """Process mesh without shapekeys using simple merge operation"""
-        logger.debug(f"Processing mesh without shapekeys: {mesh['mesh'].name}")
-        mesh["mesh"].select_set(True)
-        context.view_layer.objects.active = mesh["mesh"]
-        bpy.ops.object.mode_set(mode='EDIT')
-        mesh["mesh"].data.vertices.foreach_set("select", [False] * len(mesh["mesh"].data.vertices))
-
-        bpy.ops.mesh.select_all(action="INVERT")
-        bpy.ops.mesh.remove_doubles(threshold=merge_distance, use_unselected=False)
-        bpy.ops.object.mode_set(mode='OBJECT')
-        mesh["mesh"].select_set(False)
-
-    def finish_mesh_processing(self, context: Context, mesh: MeshEntry, advanced: bool, merge_distance: float) -> None:
-        """Complete the mesh processing by performing final merge operations"""
-        logger.debug("Finishing mesh processing")
-        mesh["mesh"].select_set(True)
-        context.view_layer.objects.active = mesh["mesh"]
-        bpy.ops.object.mode_set(mode='EDIT')
-        bpy.ops.mesh.select_all(action="INVERT")
-        bpy.ops.mesh.remove_doubles(threshold=merge_distance, use_unselected=False)
-        
-        bpy.ops.object.mode_set(mode='OBJECT')
-        mesh["mesh"].select_set(False)
-
     def modal(self, context: Context, event: Event) -> set[ModalReturnType]:
         """Modal operator execution"""
         try:
-            if not self.objects_to_do:
+            if not self.objects_to_do or len(self.objects_to_do) <= 0:
                 self.report({'INFO'}, t("Optimization.remove_doubles_completed"))
                 logger.info("Finishing modal execution of merge doubles safely")
                 return {'FINISHED'}
             
-            mesh = self.objects_to_do[0]
+            mesh: MeshEntry = self.objects_to_do.pop(0)
-            mesh_data = mesh["mesh"].data
+            merge_distance: float = self.merge_distance
-            advanced = context.scene.avatar_toolkit.remove_doubles_advanced
-            merge_distance = context.scene.avatar_toolkit.remove_doubles_merge_distance
             
-            if len(mesh['shapekeys']) > 0 and not advanced:
-                shapekeyname = mesh['shapekeys'].pop(0)
-                mesh["mesh"].active_shape_key_index = mesh_data.shape_keys.key_blocks.find(shapekeyname)
-                logger.debug(f"Processing shapekey {shapekeyname}")
-                self.modify_mesh(context, mesh)
             
-            elif not mesh_data.shape_keys:
+            #find which vertices merge on all shapekeys using bmesh, a fast way of doing it - @989onan
-                self.process_simple_mesh(context, mesh, merge_distance)
+            final_merged_vertex_group = [i for i in range(0,len(mesh['mesh'].data.vertices))]
-                self.objects_to_do.pop(0)
+            for shape in mesh["shapekeys"]:
+                select_obj(context, shape, target_mode='EDIT')
+                bmesh_mesh: bmesh.types.BMesh = bmesh.from_edit_mesh(shape.data)
+                selected_verts: list[bmesh.types.BMVert] = [vert for vert in bmesh_mesh.verts if vert.select == True]
+                i: int = 0
+                merged_vertices: set[int] = set()
+                mergers: dict[bmesh.types.BMVert, bmesh.types.BMVert]
+                for name,mergers in bmesh.ops.find_doubles(bmesh_mesh,verts=selected_verts,dist=merge_distance).items():
+                    for source_vert,target_vert in mergers.items():
+                        merged_vertices.add(source_vert.index)
+                        merged_vertices.add(target_vert.index)
                     
-            elif not (mesh["cur_vertex_pass"] > mesh["vertices"]) and advanced: #advanced merging vertex by vertex
+                    final_merged_vertex_group = [v for v in final_merged_vertex_group if v in merged_vertices]
-                if(mesh["cur_vertex_pass"] < 0): #make sure it doesn't go below 0 and explode when advancing backwards from a previous step
-                    mesh["cur_vertex_pass"] = 0
                 
-                if vertex_moves(mesh["mesh"].data, mesh["cur_vertex_pass"]): # do not do advanced merging for vertices that don't move
+            select_obj(context, mesh['mesh'], target_mode='EDIT')
-                    mesh["cur_vertex_pass"] -= self.modify_mesh_advanced(context, mesh)-2 #advance forward or backwards based on how many vertices actually got merged, changing the list size.
+            data_mesh: bpy.types.Mesh = mesh['mesh'].data
-                    #if above returns 1 (no vertices other than this one being merged to ourselves), advance by 1. else don't advance or go backwards. Makes sure all vertices get merged in the end.
+            bmesh_mesh: bmesh.types.BMesh = bmesh.from_edit_mesh(data_mesh)
-                else:
+            mergable_on_all_shapes: list[bmesh.types.BMVert] = [vert for vert in bmesh_mesh.verts if vert.index in final_merged_vertex_group]
-                    mesh["cur_vertex_pass"] += 1
             
-            elif (mesh["cur_vertex_pass"] > mesh["vertices"]) and advanced and len(mesh['shapekeys']) > 0: #after advanced merging has gone past all the moving vertices, now we need to merge non moving vertices.
+            mappings: dict[bmesh.types.BMVert,bmesh.types.BMVert] = bmesh.ops.find_doubles(bmesh_mesh,verts=mergable_on_all_shapes,dist=merge_distance)["targetmap"]
-                shapekeyname = mesh['shapekeys'].pop(0)
+
-                mesh["mesh"].active_shape_key_index = mesh_data.shape_keys.key_blocks.find(shapekeyname)
+            bmesh.ops.weld_verts(bmesh_mesh,targetmap=mappings)
-                logger.debug(f"Processing shapekey {shapekeyname}")
+            bmesh.update_edit_mesh(data_mesh, destructive=True)
-                self.modify_mesh(context, mesh)
+
-            else:
+
-                self.finish_mesh_processing(context, mesh, advanced, merge_distance)
+            for shape in mesh["shapekeys"]: 
-                self.objects_to_do.pop(0)
+                bpy.data.objects.remove(shape)
 
             return {'RUNNING_MODAL'}
             
         except Exception as e:
-            logger.error(f"Error in modal: {str(e)}")
+            logger.error(f"Error in modal: {traceback.format_exception(e)}")
             return {'CANCELLED'}

resources/translations/en_US.json

@@ -128,9 +128,7 @@
     "Optimization.combine_materials": "Combine Materials",
     "Optimization.combine_materials_desc": "Combine similar materials to reduce draw calls",
     "Optimization.remove_doubles": "Remove Doubles",
-    "Optimization.remove_doubles_desc": "Remove duplicate vertices",
+    "Optimization.remove_doubles_desc": "Remove duplicate vertices safely, keeping shapekeys preserved.",
-    "Optimization.remove_doubles_advanced": "Advanced",
-    "Optimization.remove_doubles_advanced_desc": "Remove duplicate vertices with advanced options",
     "Optimization.join_all_meshes": "Join All",
     "Optimization.join_all_meshes_desc": "Join all meshes in the scene",
     "Optimization.join_selected_meshes": "Join Selected",
@@ -158,8 +156,6 @@
     "Optimization.error.join_selected": "Failed to join selected meshes: {error}",
     "Optimization.merge_distance": "Merge Distance",
     "Optimization.merge_distance_desc": "Distance within which vertices will be merged",
-    "Optimization.remove_doubles_warning": "This process may take a long time",
-    "Optimization.remove_doubles_wait": "Blender may seem unresponsive during this operation",
     "Optimization.error.remove_doubles": "Failed to remove doubles: {error}",
     "Optimization.no_armature": "No armature selected",
     "Optimization.processing_mesh": "Processing mesh: {name}",

ui/optimization_panel.py

@@ -4,7 +4,7 @@ from bpy.types import Panel, Context, UILayout, Operator
 from .main_panel import AvatarToolKit_PT_AvatarToolkitPanel, CATEGORY_NAME
 from ..core.translations import t
 from ..functions.optimization.materials_tools import AvatarToolkit_OT_CombineMaterials
-from ..functions.optimization.remove_doubles import AvatarToolkit_OT_RemoveDoubles,AvatarToolkit_OT_RemoveDoublesAdvanced
+from ..functions.optimization.remove_doubles import AvatarToolkit_OT_RemoveDoubles
 from ..functions.optimization.mesh_tools import AvatarToolkit_OT_JoinAllMeshes, AvatarToolkit_OT_JoinSelectedMeshes
 
 class AvatarToolKit_PT_OptimizationPanel(Panel):
@@ -40,7 +40,6 @@ class AvatarToolKit_PT_OptimizationPanel(Panel):
         # Remove Doubles Row
         row: UILayout = col.row(align=True)
         row.operator(AvatarToolkit_OT_RemoveDoubles.bl_idname, icon='MESH_DATA')
-        row.operator(AvatarToolkit_OT_RemoveDoublesAdvanced.bl_idname, icon='PREFERENCES')
         
         # Join Meshes Box
         join_box: UILayout = layout.box()