Add the identity operator, it makes learning much better

ID can appear in many places in the genome, and makes "gaps" that can be
filled by other operators when mutating. This allows programs of 20
genes to produce short code, with many identities, and learn stably on
SimpleGoal

TD3_program_synthesis.py

@@ -78,7 +78,7 @@ class Args:
 
     num_generations: int = 20
     num_parents_mating: int = 20
-    mutation_probability: float = 0.1
+    mutation_probability: float = 0.05
 
 def make_env(env_id, seed, idx, capture_video, run_name):
     if capture_video and idx == 0:

optim.py

@@ -5,7 +5,7 @@ import numpy as np
 import pyrallis
 from dataclasses import dataclass
 
-from postfix_program import Program, NUM_OPERATORS, InvalidProgramException
+from postfix_program import *
 
 def print_fitness(ga, fitnesses):
     print('F', fitnesses.mean(), file=sys.stderr)
@@ -17,10 +17,8 @@ class ProgramOptimizer:
         self.low = low
         self.high = high
 
-        # Create the initial population
-        # We create it so these random programs try all the operators and read all the state variables
-        self.initial_population = np.random.random((config.num_individuals, config.num_genes))  # Random numbers between 0 and 1
-        self.initial_population *= -(NUM_OPERATORS + self.state_dim)         # Tokens between -NUM_OPERATORS - state_dim and 0
+        # Create the initial population of only identities
+        self.initial_population = (-ID_INDEX - 1 - 0.5) * np.ones((config.num_individuals, config.num_genes))
 
         self.best_solution = self.initial_population[0]
         self.best_fitness = None

postfix_program.py

@@ -13,7 +13,11 @@ class Operator:
     def __str__(self):
         return self.name
 
+ID = Operator('id', 1, lambda a: a)
 OPERATORS = [
+    ID,
+    ID,
+    ID,
     Operator('reciprocal', 1, lambda a: 1 / a if abs(a) > 0.05 else 20.0 * sgn(a)),
     Operator('exp', 1, lambda a: math.exp(min(a, 10.0))),
     Operator('trunc', 1, lambda a: float(int(a))),
@@ -21,21 +25,31 @@ OPERATORS = [
     Operator('sin', 1, lambda a: math.sin(a)),
     Operator('sqrt', 1, lambda a: math.sqrt(a) if a >= 0.0 else 0.0),
     Operator('cos', 1, lambda a: math.cos(a)),
+    ID,
+    ID,
+    ID,
     Operator('neg', 1, lambda a: -a),
     Operator('-abs', 1, lambda a: -abs(a)),
     Operator('-sin', 1, lambda a: -math.sin(a)),
     Operator('-sqrt', 1, lambda a: -math.sqrt(a) if a >= 0.0 else 0.0),
     Operator('-cos', 1, lambda a: -math.cos(a)),
     Operator('-exp', 1, lambda a: -math.exp(min(a, 10.0))),
+    ID,
+    ID,
+    ID,
 
     Operator('min', 2, lambda a, b: min(a, b)),
     Operator('max', 2, lambda a, b: max(a, b)),
     Operator('+', 2, lambda a, b: a + b),
     Operator('*', 2, lambda a, b: a * b),
+    ID,
+    ID,
+    ID,
 
     Operator('select', 3, lambda a, iftrue, iffalse: iftrue if a > 0 else iffalse),
 ]
 NUM_OPERATORS = len(OPERATORS)
+ID_INDEX = [i for i in range(NUM_OPERATORS) if OPERATORS[i] is ID][0]
 
 class InvalidProgramException(Exception):
     pass
@@ -53,7 +67,10 @@ class Program:
 
         def on_operator_func(stack, operator, operands):
             # Put a string representation of the operator on the stack
-            if len(operands) == 1:
+            if operator.name == 'id':
+                # Identity
+                result = operands[0]
+            elif len(operands) == 1:
                 result = f"{operator.name}({operands[0]})"
             elif operator.name in ['min', 'max']:
                 # two-operand operator that is a function call