Fix: better dependency resolve in Schema parse

Before this commit, some code in schema_solve was catching dependencies that hadn't
been discovered or prepared. Now they are discovered ahead of time by a more robust
search -- however the fallback code is still there just in case.

Further work:
 - refactor the parse code to make it clearer
 - profile with complex cases to see if further optimization is needed
 - text complex edge cases and rare node combinations.

__init__.py

@@ -11,13 +11,12 @@ from . import ( ops_nodegroup,
                 schema_definitions,
               )
 from .ops_generate_tree import GenerateMantisTree
-from bpy.types import NodeSocket
 
 from .utilities import prRed
 
 MANTIS_VERSION_MAJOR=0
 MANTIS_VERSION_MINOR=9
-MANTIS_VERSION_SUB=14
+MANTIS_VERSION_SUB=15
 
 classLists = [module.TellClasses() for module in [
  link_definitions,
@@ -39,7 +38,6 @@ while (classLists):
     classes.extend(classLists.pop())
 
 interface_classes = []
-from bpy import app
 
 import nodeitems_utils
 from nodeitems_utils import NodeCategory, NodeItem

blender_manifest.toml

@@ -3,7 +3,7 @@ schema_version = "1.0.0"
 # Example of manifest file for a Blender extension
 # Change the values according to your extension
 id = "mantis"
-version = "0.9.14"
+version = "0.9.15"
 name = "Mantis"
 tagline = "Mantis is a rigging nodes toolkit"
 maintainer = "Nodespaghetti <josephbburg@protonmail.com>"

readtree.py

@@ -272,98 +272,112 @@ def solve_schema_to_tree(nc, all_nc, roots=[]):
 #                                  PARSE NODE TREE                                  #
 # *** # *** # *** # *** # *** # *** # *** # *** # *** # *** # *** # *** # *** # *** #
 
+schema_bl_idnames = [   "SchemaIndex",
+                        "SchemaArrayInput",
+                        "SchemaArrayInputGet",
+                        "SchemaArrayOutput",
+                        "SchemaConstInput",
+                        "SchemaConstOutput",
+                        "SchemaOutgoingConnection",
+                        "SchemaIncomingConnection", 
+                    ]
+
 from .utilities import get_all_dependencies
-def get_schema_length_dependencies(node):
-    """ Find all of the nodes that the Schema Length input depends on. """
-    # the deps recursively from the from_nodes connected to Schema Length
+def get_schema_length_dependencies(node, all_nodes={}):
+    """ Get a list of all dependencies for the given node's length or array properties.
+        This function will also recursively search for dependencies in its sub-trees.
+    """
     deps = []
-    # return get_all_dependencies(node)
-    inp = node.inputs.get("Schema Length")
-    if not inp:
-        inp = node.inputs.get("Array")
+    prepare_links_to = ['Schema Length','Array', 'Index']
+    def extend_dependencies_from_inputs(node):
+        for inp in node.inputs.values():
+            for l in inp.links:
+                if "MANTIS_AUTOGENERATED" in l.from_node.signature: 
+                    deps.extend([l.from_node]) # why we need this lol
+                if inp in prepare_links_to: 
+                    deps.extend(get_all_dependencies(l.from_node))
+    def deps_filter(dep): # remove any nodes inside the schema
+        if len(dep.signature) > len(node.signature):
+            for i in range(len(node.signature)):
+                dep_sig_elem, node_sig_elem = dep.signature[i], node.signature[i]
+                if dep_sig_elem != node_sig_elem: break # they don't match, it isn't an inner-node
+            else: # remove this, it didn't break, meaning it shares signature with outer node
+                return False # this is an inner-node
+            return True
+        pass
     # this way we can handle Schema and Array Get nodes with one function
-    # ... since I may add more in the future this is not a robust solution HACK
-    for l in inp.links:
-        deps.extend(get_all_dependencies(l.from_node))
-    if inp := node.inputs.get("Index"):
-        for l in inp.links:
-            deps.extend(get_all_dependencies(l.from_node))
-    # now get the auto-generated simple inputs. These should not really be there but I haven't figured out how to set things directly yet lol
-    for inp in node.inputs.values():
-        for l in inp.links:
-            if "MANTIS_AUTOGENERATED" in l.from_node.signature:
-                # l.from_node.bPrepare() # try this...
-                # l.from_node.prepared = True; l.from_node.executed = True
-                deps.extend([l.from_node]) # why we need this lol
-
-    return deps
+    extend_dependencies_from_inputs(node)
+    if node.node_type == 'DUMMY_SCHEMA':
+        trees = [(node.prototype.node_tree, node.signature)] # this is UI data
+        while trees:
+            tree, tree_signature = trees.pop()
+            print(tree_signature)
+            for sub_ui_node in tree.nodes:
+                if sub_ui_node.bl_idname in ['NodeReroute', 'NodeFrame']:
+                    continue
+                if sub_ui_node.bl_idname in schema_bl_idnames:
+                    sub_node = all_nodes[(*tree_signature, sub_ui_node.bl_idname)]
+                else:
+                    sub_node = all_nodes[(*tree_signature, sub_ui_node.name)]
+                if sub_node.node_type == 'DUMMY_SCHEMA':
+                    extend_dependencies_from_inputs(sub_node)
+                    trees.append((sub_node.prototype.node_tree, sub_node.signature))
+    
+    filtered_deps = filter(deps_filter, deps)
+
+
+    return list(filtered_deps)
 
-        
-def parse_tree(base_tree):
-    from uuid import uuid4 # do this here?
-    base_tree.execution_id = uuid4().__str__() # set this, it may be used by nodes during execution
 
+def parse_tree(base_tree):
+    from uuid import uuid4
+    base_tree.execution_id = uuid4().__str__() # set the unique id of this execution
+   
+    import time
+    data_start_time = time.time()
     # annoyingly I have to pass in values for all of the dicts because if I initialize them in the function call
     #  then they stick around because the function definition inits them once and keeps a reference
     # so instead I have to supply them to avoid ugly code or bugs elsewhere
-    # it's REALLY confusing when you run into this sort of problem. So it warrants four entire lines of comments!         
-    import time
-
-    data_start_time = time.time()
-
-    dummy_nodes, all_nc, all_schema =  data_from_tree(base_tree, tree_path = [None], dummy_nodes = {}, all_nc = {}, all_schema={})
-
-    # return
-
-    prGreen(f"Pulling data from tree took {time.time() - data_start_time} seconds")
-
-    for sig, dummy in dummy_nodes.items():
+    # it's REALLY confusing when you run into this sort of problem. So it warrants four entire lines of comments!      
+    dummy_nodes, all_mantis_nodes, all_schema =  data_from_tree(base_tree, tree_path = [None], dummy_nodes = {}, all_nc = {}, all_schema={})
+    for dummy in dummy_nodes.values():    # reroute the links in the group nodes
         if (hasattr(dummy, "reroute_links")):
-            dummy.reroute_links(dummy, all_nc)
+            dummy.reroute_links(dummy, all_mantis_nodes)
+    prGreen(f"Pulling data from tree took {time.time() - data_start_time} seconds")
     
-    # TODO
-    # MODIFY BELOW to use hierarchy_dependencies instead
-    # SCHEMA DUMMY nodes will need to gather the hierarchy and non-hierarchy dependencies
-    # so SCHEMA DUMMY will not make their dependencies all hierarchy
-    # since they will need to be able to send drivers and such   
     start_time = time.time()
-
-
-    sig_check = (None, 'Node Group.001', 'switch_thigh')
-    roots = []
-    arrays = []
+    roots, array_nodes = [], []
     from .misc_containers import UtilityArrayGet
-    for nc in all_nc.values():
+    for mantis_node in all_mantis_nodes.values():
         # clean up the groups
-        if nc.node_type in ["DUMMY"]:
-            if nc.prototype.bl_idname in ("MantisNodeGroup", "NodeGroupOutput"):
+        if mantis_node.node_type in ["DUMMY"]:
+            if mantis_node.prototype.bl_idname in ("MantisNodeGroup", "NodeGroupOutput"):
                 continue
-        
-        from .base_definitions import from_name_filter, to_name_filter
-
-        init_dependencies(nc)
-        init_connections(nc)
-        check_and_add_root(nc, roots, include_non_hierarchy=True)
-        if isinstance(nc, UtilityArrayGet):
-            arrays.append(nc)
+        # Initialize the dependencies and connections (from/to links) for each node.
+        # we record & store it because using a getter is much slower (according to profiling)
+        init_dependencies(mantis_node); init_connections(mantis_node)
+        check_and_add_root(mantis_node, roots, include_non_hierarchy=True)
+        # Array nodes need a little special treatment, they're quasi-schemas
+        if isinstance(mantis_node, UtilityArrayGet):
+            array_nodes.append(mantis_node)
 
     from collections import deque
     unsolved_schema = deque()
     solve_only_these = []; solve_only_these.extend(list(all_schema.values()))
     for schema in all_schema.values():
-        # so basically we need to check every parent node if it is a schema
-        # this is a fairly slapdash solution but it works and I won't change it
+        # We can remove the schema that are inside another schema tree.
         for i in range(len(schema.signature)-1): # -1, we don't want to check this node, obviously
             if parent := all_schema.get(schema.signature[:i+1]):
+                # This will be solved along with its parent schema.
                 solve_only_these.remove(schema)
                 break
         else:
-            init_schema_dependencies(schema, all_nc)
-            solve_only_these.extend(get_schema_length_dependencies(schema))
+            init_schema_dependencies(schema, all_mantis_nodes)
+            solve_only_these.extend(get_schema_length_dependencies(schema, all_mantis_nodes))
             unsolved_schema.append(schema)
-    for array in arrays:
+    for array in array_nodes:
         solve_only_these.extend(get_schema_length_dependencies(array))
-    solve_only_these.extend(arrays)
+    solve_only_these.extend(array_nodes)
     schema_solve_done = set()
 
     solve_only_these = set(solve_only_these)
@@ -385,7 +399,7 @@ def parse_tree(base_tree):
                     solve_layer.appendleft(n)
                     break
             else:
-                solved_nodes = solve_schema_to_tree(n, all_nc, roots)
+                solved_nodes = solve_schema_to_tree(n, all_mantis_nodes, roots)
                 unsolved_schema.remove(n)
                 schema_solve_done.add(n)
                 for node in solved_nodes.values():
@@ -411,11 +425,11 @@ def parse_tree(base_tree):
         raise RuntimeError("Failed to resolve all schema declarations")
     # I had a problem with this looping forever. I think it is resolved... but I don't know lol
 
-    all_nc = list(all_nc.values()).copy()
+    all_mantis_nodes = list(all_mantis_nodes.values()).copy()
     kept_nc = {}
-    while (all_nc):
-        nc = all_nc.pop()
-        if nc in arrays:
+    while (all_mantis_nodes):
+        nc = all_mantis_nodes.pop()
+        if nc in array_nodes:
             continue
 
         if nc.node_type in ["DUMMY"]:

schema_solve.py

@@ -12,7 +12,7 @@ from .node_container_common import setup_custom_props_from_np
 
 
 class SchemaSolver:
-    def __init__(self, schema_dummy, nodes, prototype, signature=None):
+    def __init__(self, schema_dummy, nodes, prototype, signature=None,):
         self.all_nodes = nodes # this is the parsed tree from Mantis
         self.node = schema_dummy
         self.tree = prototype.node_tree
@@ -37,7 +37,6 @@ class SchemaSolver:
         self.index_link = self.node.inputs['Schema Length'].links[0]
         self.solve_length = self.node.evaluate_input("Schema Length")
         # I'm making this a property of the solver because the solver's data is modified as it solves each iteration
-        # So
         self.index = 0
 
         self.init_schema_links()
@@ -320,7 +319,22 @@ class SchemaSolver:
         connection = incoming.from_node.outputs[incoming.from_socket].connect(node=to_node, socket=ui_link.to_socket.name)
         init_connections(incoming.from_node)
                     
-
+    def prepare_nodes(self, unprepared):
+        # At this point, we've already run a pretty exhaustive preperation phase to prep the schema's dependencies
+        # So we should not need to add any new dependencies unless there is a bug elsewhere.
+        # and in fact, I could skip this in some cases, and should investigate if profiling reveals a slowdown here.
+        while unprepared:
+            nc = unprepared.pop()
+            if sum([dep.prepared for dep in nc.hierarchy_dependencies]) == len(nc.hierarchy_dependencies):
+                nc.bPrepare()
+                if nc.node_type == 'DUMMY_SCHEMA':
+                    self.solve_nested_schema(nc)
+            else: # Keeping this for-loop as a fallback, it should never add dependencies though
+                for dep in nc.hierarchy_dependencies:
+                    if not dep.prepared and dep not in unprepared:
+                        prOrange(f"Adding dependency... {dep}")
+                        unprepared.appendleft(dep)
+                unprepared.appendleft(nc) # just rotate them until they are ready.
 
     def solve_iteration(self):
         """ Solve an iteration of the schema.
@@ -418,21 +432,7 @@ class SchemaSolver:
             self.solved_nodes[k]=v
             init_dependencies(v) # it is hard to overstate how important this single line of code is
         
-        # This while-loop is a little scary, but in my testing it has never been a problem.
-        # At this point, we've already run a pretty exhaustive preperation phase to prep the schema's dependencies
-        # So at this point, if this while loop hangs it is because of an error elsewhere.
-        while unprepared:
-            nc = unprepared.pop()
-            if sum([dep.prepared for dep in nc.hierarchy_dependencies]) == len(nc.hierarchy_dependencies):
-                nc.bPrepare()
-                if nc.node_type == 'DUMMY_SCHEMA':
-                    self.solve_nested_schema(nc)
-            else: #TODO find out what I am missing elsewhere that makes this necessary
-                for dep in nc.hierarchy_dependencies:
-                    if not dep.prepared and dep not in unprepared:
-                        unprepared.appendleft(dep)
-                #TODO
-                unprepared.appendleft(nc) # just rotate them until they are ready.
+        self.prepare_nodes(unprepared)
         
         while(awaiting_prep_stage):
             ui_link = awaiting_prep_stage.pop()