New Feature: Overlay Variables

This commit adds the Overlay Variables feature to Mantis. Overlays are
a way of passing data from parent node trees throughout the tree, so
that a node in Mantis can request the value of a group-node input from
any group node in its hierarchy. The "newest" version of the Overlay
is always used.

base_definitions.py

@@ -707,22 +707,10 @@ def can_remove_socket_for_autogen(node, socket) -> bool:
         return True # HACK because most utilities don't have socket templates yet
     return False
 
-# TODO:
-#   - get the execution context in the execution code
-#   - from there, begin to use it for stuff I can't do without it
-#   - and slowly start transferring stuff to it
-
-# The Mantis Overlay class is used to store node-tree specific information
-#  such as inputs and outputs
-# used for e.g. allowing strings to pass as $variables in node names
-class MantisOverlay():
-    def __init__( self, parent, inputs, outputs, ):
-        pass
-
 # The MantisExecutionContext class is used to store the execution-specific variables
 # that are used when executing the tree
 # Importantly, it is NOT used to store variables between solutions, these belong to the
-#   tree itself.
+#   tree class.
 class MantisExecutionContext():
     def __init__(
             self,
@@ -731,7 +719,10 @@ class MantisExecutionContext():
         self.base_tree = base_tree
         self.execution_id = base_tree.execution_id
         self.execution_failed=False
+        self.interface_nodes = {}
         self.b_objects={} # objects created by Mantis during execution
+        self.tagged_for_overlay = [] # nodes mutated by overlay system
+        self.overlay_data = [] # dict of list of Overlay objects
 
 from typing import Any
 
@@ -853,6 +844,13 @@ class MantisNode:
         change_handled=False
         if socket_name is None: socket_name = ui_socket.name
         value = ui_socket.default_value
+        # apply overlay variables here so this updates predictably
+        if (isinstance(value, str) and
+                (self.signature, ui_socket.identifier) in self.mContext.tagged_for_overlay):
+            from .readtree import node_scan_overlays
+            value = node_scan_overlays(self, ui_socket.identifier)
+        # TODO: I need to make Node Groups apply overlays downstream, but since
+        #       they don't update live anyway, no need just yet
         if socket_name == 'Enable': value = not value
         try:
             self.parameters[ui_socket.name]=value
@@ -925,6 +923,9 @@ class MantisNode:
                 # fill these because they are probably Input nodes.
                 elif hasattr(node_socket, "default_value"):
                     if (value := get_socket_value(node_socket)) is not None:
+                        if isinstance(value, str) and "$" in value:
+                            from .utilities import handle_overlay_detect
+                            value = handle_overlay_detect(self, node_socket.identifier, value)
                         self.parameters[key] = value
                     else:
                         raise RuntimeError(wrapRed("No value found for " + self.__repr__() + " when filling out node parameters for " + ui_node.name + "::"+node_socket.name))

internal_nodes.py

@@ -63,7 +63,7 @@ class AutoGenNode(MantisNode):
 class GroupInterface(MantisNode):
     def __init__(self, signature, base_tree, ui_node, in_out):
         super().__init__(signature, base_tree)
-        self.node_type = 'UTILITY'
+        self.node_type = 'INTERFACE'
         self.prepared, self.executed = True, True; sockets = []
         self.in_out = in_out
         # init the sockets based on in/out, then set up traversal

readtree.py

@@ -141,7 +141,7 @@ def make_connections_to_ng_dummy(base_tree, tree_path_names, local_mantis_nodes,
             else:
                 prRed("No available auto-generated class for input %s in %s" % (inp.name, np.name))
 
-def gen_mantis_nodes(base_tree, current_tree, tree_path_names, all_mantis_nodes, local_mantis_nodes, dummy_nodes, group_nodes, schema_nodes ):
+def gen_mantis_nodes(base_tree, current_tree, tree_path_names, mContext, all_mantis_nodes, local_mantis_nodes, dummy_nodes, group_nodes, schema_nodes ):
     from .internal_nodes import DummyNode
     for ui_node in current_tree.nodes:
         # HACK I found that this isn't being set sometimes. I wonder why? It makes the most sense to do this here.
@@ -182,10 +182,11 @@ def gen_mantis_nodes(base_tree, current_tree, tree_path_names, all_mantis_nodes,
             mantis_node = None
             prRed(f"Can't make mantis_node for.. {ui_node.bl_idname}")
         # this should be done at init
+        mantis_node.mContext = mContext
         if mantis_node.signature[0] not in ['MANTIS_AUTOGENERATED'] and mantis_node.node_type not in ['SCHEMA', 'DUMMY', 'DUMMY_SCHEMA']:
             mantis_node.fill_parameters()
 
-def data_from_tree(base_tree, tree_path, dummy_nodes, all_mantis_nodes, all_schema):#
+def data_from_tree(base_tree, tree_path, mContext, dummy_nodes, all_mantis_nodes, all_schema):#
     # TODO: it should be relatively easy to make this use a while loop instead of recursion.
     local_mantis_nodes, group_nodes = {}, []
     tree_path_names = [tree.name for tree in tree_path if hasattr(tree, "name")]
@@ -197,7 +198,7 @@ def data_from_tree(base_tree, tree_path, dummy_nodes, all_mantis_nodes, all_sche
     if current_tree: # the node-group may not have a tree set - if so, ignore it.
         from .utilities import clear_reroutes
         links = clear_reroutes(list(current_tree.links))
-        gen_mantis_nodes(base_tree, current_tree, tree_path_names, all_mantis_nodes, local_mantis_nodes, dummy_nodes, group_nodes, all_schema)
+        gen_mantis_nodes(base_tree, current_tree, tree_path_names, mContext, all_mantis_nodes, local_mantis_nodes, dummy_nodes, group_nodes, all_schema)
 
         from .utilities import link_mantis_nodes
         for link in links:
@@ -207,9 +208,94 @@ def data_from_tree(base_tree, tree_path, dummy_nodes, all_mantis_nodes, all_sche
             insert_default_values_base_tree(base_tree, all_mantis_nodes)
         # Now, descend into the Node Groups and recurse
         for mantis_node in group_nodes:
-            data_from_tree(base_tree, tree_path+[mantis_node.ui_node], dummy_nodes, all_mantis_nodes, all_schema)
+            data_from_tree(base_tree, tree_path+[mantis_node.ui_node], mContext, dummy_nodes, all_mantis_nodes, all_schema)
     return dummy_nodes, all_mantis_nodes, all_schema
 
+# Do not use these input names as overlay keys:
+forbidden_key_names = ["",]
+forbidden_in_key_name = ['$',]
+def node_scan_overlays(node, socket_id,):
+    interface_nodes = node.mContext.interface_nodes
+    from .utilities import get_ui_node
+    ui_node = get_ui_node(node.ui_signature, node.base_tree)
+    # we'll assume that the overlay tag system checked if this is linked.
+    # First let's check if it is a simple input node, by checking outputs that are marked "input"
+    for socket in ui_node.outputs:
+        # weird edge case for simple input nodes with unnamed sockets
+        # the socket.identifier is not used by Mantis but it defaults to 'socket'\
+        if socket.identifier == socket_id and socket.input == True: break
+    else:
+        for socket in ui_node.inputs:
+            if socket.identifier == socket_id: break
+        else:
+            raise KeyError(f"Socket not found by ID {socket_id} in node {node.ui_signature}\n"
+                        f"Candidates: {[socket.identifier for socket in ui_node.inputs]}")
+    value = socket.default_value # get the ACTUAL UI value
+    if not isinstance(value, str): return
+    # First step: get the interface nodes and the overlay keys
+    # this should only check interfaces that are ancestors of this node
+    my_interfaces = []
+    for index in range(len(node.signature)-1):
+        if index == 0: continue # root
+        interface_signature = (*node.signature[:index+1], "InputInterface")
+        my_interfaces.append( interface_nodes[interface_signature] )
+    # the my_interfaces list is sorted in REVERSE order of importance
+    # Next, get all the overlay providers.
+    name_to_id_map = {}
+    overlay_keys = set()
+    root_overlay_values = {}
+    # First, we'll get the base tree's default inputs
+    for interface_item in node.mContext.base_tree.interface.items_tree:
+        if interface_item.item_type != "SOCKET": continue
+        if interface_item.in_out != "INPUT": continue
+        # NOTE this will need to change if overlays extend to other data types
+        if not hasattr(interface_item, "default_string"): continue
+        # NOTE that there may be duplicate names. Too bad!
+        root_overlay_values[interface_item.name] = interface_item.default_string
+        overlay_keys.add(interface_item.name)
+        name_to_id_map[interface_item.name] = None # don't worry about it
+    for interface_node in interface_nodes.values():
+        # problem: the interface node stores socket identifiers as keys
+        # so in reality we need to get the group UI node and ask for its input names
+        # if there is a duplicate name, too bad!
+        group_ui_node = get_ui_node(interface_node.ui_signature[:-1], node.base_tree)
+        for input in group_ui_node.inputs:
+            if input.name in forbidden_key_names: continue
+            for substr in forbidden_in_key_name:
+                if substr in input.name:
+                    break
+            else: # if we do not find a bad substr
+                name_to_id_map[input.name]=input.identifier
+                overlay_keys.add(input.name)
+    # sort by reverse length so we find "CatsAndFrogs" before "Cats"
+    overlay_keys = sorted(overlay_keys, key= lambda a :-1*len(a))
+    overlay_tries = value.split("$")
+    result = overlay_tries[0] # the 0th element is always before the split character
+    # the reason for the difference between overlay_try and key is because
+    #  we don't actually know if the $text part has a key in it
+    for overlay_try in overlay_tries[1:]:
+        overlay_resolved_value = "$" + overlay_try
+        for key in overlay_keys:
+            if key in overlay_try:
+                overlay_resolved_value = "$" + key
+                for interface in reversed(my_interfaces): # since this is reversed when we build it
+                    if interface.inputs.get(actual_key := name_to_id_map.get(key)):
+                        overlay_resolved_value = overlay_try.replace(key, interface.evaluate_input(actual_key))
+                        break
+                else: # we'll try the base tree
+                    if base_tree_result := root_overlay_values.get(overlay_try):
+                        overlay_resolved_value = base_tree_result
+                    else:
+                        prOrange(f"WARN: no overlay value applied for {node.ui_signature} for key {key}")
+                break # we found the key and are done here. One found Key per overlay-try
+        result += overlay_resolved_value
+    # Simple Input sockets with unnamed sockets assign this ID automatically
+    # But Mantis uses the name instead. Usually the name and ID are the same.
+    if socket_id == 'socket' and ui_node.outputs[0].name == "":
+        socket_id = ''
+    node.parameters[socket_id] = result
+    return result # because sometimes I need to get the result, too
+
 from .utilities import check_and_add_root, init_connections, init_dependencies, init_schema_dependencies
 
 def is_signature_in_other_signature(parent_signature, child_signature):
@@ -380,7 +466,7 @@ def parse_tree(base_tree, error_popups=False):
     #  then they stick around because the function definition inits them once and keeps a reference
     # so instead I have to supply them to avoid ugly code or bugs elsewhere
     # it's REALLY confusing when you run into this sort of problem. So it warrants four entire lines of comments!      
-    dummy_nodes, all_mantis_nodes, all_schema =  data_from_tree(base_tree, tree_path = [None], dummy_nodes = {}, all_mantis_nodes = {}, all_schema={})
+    dummy_nodes, all_mantis_nodes, all_schema =  data_from_tree(base_tree, tree_path = [None], mContext=mContext, dummy_nodes = {}, all_mantis_nodes = {}, all_schema={})
     for dummy in dummy_nodes.values():    # reroute the links in the group nodes
         if (hasattr(dummy, "reroute_links")):
             dummy.reroute_links(dummy, all_mantis_nodes)
@@ -394,7 +480,8 @@ def parse_tree(base_tree, error_popups=False):
 
     from .base_definitions import array_output_types, GraphError
     for mantis_node in all_mantis_nodes.values():
-        # add the Mantis Context here, so that it available during parsing.
+        # TODO I still need to assign mContext here because of autogen_node
+        # so I have to do a deeper refactor to ensure autogen nodes get this at creation time.
         mantis_node.mContext = mContext
         if mantis_node.node_type in ["DUMMY"]: # clean up the groups
             if mantis_node.ui_node.bl_idname in ("MantisNodeGroup", "NodeGroupOutput"):
@@ -407,6 +494,8 @@ def parse_tree(base_tree, error_popups=False):
         if mantis_node.__class__.__name__ in array_output_types:
             solve_only_these.append(mantis_node)
             array_nodes.append(mantis_node)
+        if mantis_node.node_type == 'INTERFACE':
+            mContext.interface_nodes[mantis_node.signature] = mantis_node
 
     from itertools import chain
     for schema in chain(all_schema.values(), array_nodes):
@@ -599,6 +688,22 @@ def execute_tree(nodes, base_tree, context, error_popups = False, profile=False)
         mantis_node.reset_execution()
     mContext.execution_failed = False
 
+    interface_nodes = {}
+    from .internal_nodes import GroupInterface
+    for node in nodes.values():
+        if isinstance(node, GroupInterface):
+            interface_nodes[node.signature] = node
+    overlay_tagged_nodes = {}
+    for node_signature, socket_id in mContext.tagged_for_overlay:
+        if overlay_tagged_nodes.get(node_signature):
+            overlay_tagged_nodes[node_signature].append(socket_id)
+        else:
+            overlay_tagged_nodes[node_signature] = [socket_id]
+    def check_and_apply_overlay(n):
+        if overlays_for_this_node := overlay_tagged_nodes.get(n.signature):
+            for socket_id in overlays_for_this_node:
+                node_scan_overlays(n, socket_id,)
+
     select_me, switch_me = [], [] # switch the mode on these objects
     active = None # only need it for switching modes
     select_me = []
@@ -612,6 +717,7 @@ def execute_tree(nodes, base_tree, context, error_popups = False, profile=False)
         sorted_nodes, execution_failed = sort_execution(nodes)
         for n in sorted_nodes:
             try:
+                check_and_apply_overlay(n)
                 if not n.prepared:
                     if profiler: profiler.record(n, "bPrepare", context)
                     else: n.bPrepare(context)
@@ -636,6 +742,7 @@ def execute_tree(nodes, base_tree, context, error_popups = False, profile=False)
 
         for n in sorted_nodes:
             try:
+                check_and_apply_overlay(n)
                 if not n.prepared:
                     if profiler: profiler.record(n, "bPrepare", context)
                     else: n.bPrepare(context)

utilities.py

@@ -940,93 +940,25 @@ def SugiyamaGraph(tree, iterations):
 def project_point_to_plane(point, origin, normal):
     return point - normal.dot(point- origin)*normal
 
-##################################################################################################
-# stuff I should probably refactor!!
-##################################################################################################
-
-# This is really, really stupid way to do this
-def gen_mantis_node_input_for_data(socket):
-    # Class List #TODO deduplicate
-    from . import xForm_nodes, link_nodes, misc_nodes, primitives_nodes, deformer_nodes, math_nodes, schema_nodes
-    from .internal_nodes import NoOpNode
-    classes = {}
-    for module in [xForm_nodes, link_nodes, misc_nodes, primitives_nodes, deformer_nodes, math_nodes, schema_nodes]:
-        for cls in module.TellClasses():
-            classes[cls.__name__] = cls
-    #
-    socket_class_map = {
-                        "MatrixSocket"                         : classes["InputMatrix"],
-                        "xFormSocket"                          : None,
-                        "RelationshipSocket"                   : NoOpNode,
-                        "DeformerSocket"                       : NoOpNode,
-                        "GeometrySocket"                       : classes["InputExistingGeometryData"],
-                        "EnableSocket"                         : classes["InputBoolean"],
-                        "HideSocket"                           : classes["InputBoolean"],
-                        #
-                        "DriverSocket"                         : None,
-                        "DriverVariableSocket"                 : None, 
-                        "FCurveSocket"                         : None, 
-                        "KeyframeSocket"                       : None,
-                        "BoneCollectionSocket"                 : classes["InputString"],
-                        #
-                        "xFormParameterSocket"                 : None,
-                        "ParameterBoolSocket"                  : classes["InputBoolean"],
-                        "ParameterIntSocket"                   : classes["InputFloat"],  #TODO: make an Int node for this
-                        "ParameterFloatSocket"                 : classes["InputFloat"],
-                        "ParameterVectorSocket"                : classes["InputVector"],
-                        "ParameterStringSocket"                : classes["InputString"],
-                        #
-                        "TransformSpaceSocket"                 : classes["InputTransformSpace"],
-                        "BooleanSocket"                        : classes["InputBoolean"],
-                        "BooleanThreeTupleSocket"              : classes["InputBooleanThreeTuple"],
-                        "RotationOrderSocket"                  : classes["InputRotationOrder"],
-                        "QuaternionSocket"                     : None,
-                        "QuaternionSocketAA"                   : None,
-                        "UnsignedIntSocket"                    : classes["InputFloat"],
-                        "IntSocket"                            : classes["InputFloat"],
-                        "StringSocket"                         : classes["InputString"],
-                        #
-                        "BoolUpdateParentNode"                 : classes["InputBoolean"],
-                        "IKChainLengthSocket"                  : classes["InputFloat"],
-                        "EnumInheritScale"                     : classes["InputString"],
-                        "EnumRotationMix"                      : classes["InputString"],
-                        "EnumRotationMixCopyTransforms"        : classes["InputString"],
-                        "EnumMaintainVolumeStretchTo"          : classes["InputString"],
-                        "EnumRotationStretchTo"                : classes["InputString"],
-                        "EnumTrackAxis"                        : classes["InputString"],
-                        "EnumUpAxis"                           : classes["InputString"],
-                        "EnumLockAxis"                         : classes["InputString"],
-                        "EnumLimitMode"                        : classes["InputString"],
-                        "EnumYScaleMode"                       : classes["InputString"],
-                        "EnumXZScaleMode"                      : classes["InputString"],
-                        "EnumCurveSocket"                      : classes["InputString"],
-                        "EnumMetaRigSocket"                    : classes["InputString"],
-                        # Deformers
-                        "EnumSkinning"                         : classes["InputString"],
-                        #
-                        "FloatSocket"                          : classes["InputFloat"],
-                        "FloatFactorSocket"                    : classes["InputFloat"],
-                        "FloatPositiveSocket"                  : classes["InputFloat"],
-                        "FloatAngleSocket"                     : classes["InputFloat"],
-                        "VectorSocket"                         : classes["InputVector"],
-                        "VectorEulerSocket"                    : classes["InputVector"],
-                        "VectorTranslationSocket"              : classes["InputVector"],
-                        "VectorScaleSocket"                    : classes["InputVector"],
-                        # Drivers             
-                        "EnumDriverVariableType"               : classes["InputString"],
-                        "EnumDriverVariableEvaluationSpace"    : classes["InputString"],
-                        "EnumDriverRotationMode"               : classes["InputString"],
-                        "EnumDriverType"                       : classes["InputString"],
-                        "EnumKeyframeInterpTypeSocket"         : classes["InputString"],
-                        "EnumKeyframeBezierHandleTypeSocket"   : classes["InputString"],
-                        # Math
-                        "MathFloatOperation"                   : classes["InputString"],
-                        "MathVectorOperation"                  : classes["InputString"],
-                        "MatrixTransformOperation"             : classes["InputString"],
-                        # Schema
-                        "WildcardSocket"                       : None,
-                       }
-    return socket_class_map.get(socket.bl_idname, None)
+# If a $ character is in a value, it is an overlay request
+# in the root tree, supply it immediately, otherwise tag this node
+# once we finish pulling data from the tree we handle the overlay.
+def handle_overlay_detect(node, socket_id, value) -> str:
+    mContext = node.mContext
+    root_tree = mContext.base_tree
+    root_items_tree = root_tree.interface.items_tree
+    # detect if it is in the root tree, if so, handle immediately
+    out_value = value
+    if node.signature[0] is None and len(node.signature) ==2:
+        for key, interface_item in root_items_tree.items():
+            if interface_item.item_type != "SOCKET": continue
+            if interface_item.in_out != "INPUT": continue
+            # NOTE this will need to change if overlays extend to other data types
+            if not hasattr(interface_item, "default_string"): continue
+            out_value = out_value.replace("$"+key, interface_item.default_string)
+    else: # since it is not root, we have to tag it.
+        mContext.tagged_for_overlay.append((node.signature, socket_id))
+    return out_value
 
 ####################################
 # CURVE STUFF