Upload 57 files

LLMatic-main/.gitignore

@@ -0,0 +1,9 @@
+# Ignore python compile stuff
+*.pyc
+*.dat
+
+data
+outputs
+multirun
+logs
+rough

LLMatic-main/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LLMatic-main/README.md

@@ -0,0 +1,35 @@
+# LLMatic
+
+### Abstract:
+_Large Language Models (LLMs) have emerged as powerful tools capable of accomplishing a broad spectrum of tasks. Their abilities span numerous areas, and one area where they have made a significant impact is in the domain of code generation. Here, we propose using the coding abilities of LLMs to introduce meaningful variations to code defining neural networks. Meanwhile, Quality-Diversity (QD) algorithms are known to discover diverse and robust solutions. By merging the code-generating abilities of LLMs with the diversity and robustness of QD solutions, we introduce \texttt{LLMatic}, a Neural Architecture Search (NAS) algorithm. While LLMs struggle to conduct NAS directly through prompts, \texttt{LLMatic} uses a procedural approach, leveraging QD for prompts and network architecture to create diverse and high-performing networks. We test \texttt{LLMatic} on the CIFAR-10 and NAS-bench-201 benchmarks, demonstrating that it can produce competitive networks while evaluating just $2,000$ candidates, even without prior knowledge of the benchmark domain or exposure to any previous top-performing models for the benchmark._
+
+### To run experiments:
+
+Clone this repository:
+
+```git clone https://github.com/umair-nasir14/LLMatic.git```
+
+Install all dependencies:
+
+```
+cd LLMatic
+conda env create -f environment.yaml
+conda activate llmatic
+```
+
+Run:
+
+```python llmatic.py```
+
+All configs are present in `conf/config.py`.
+
+### Cite:
+
+```
+@article{nasir2023llmatic,
+  title={Llmatic: Neural architecture search via large language models and quality-diversity optimization},
+  author={Nasir, Muhammad U and Earle, Sam and Togelius, Julian and James, Steven and Cleghorn, Christopher},
+  journal={arXiv preprint arXiv:2306.01102},
+  year={2023}
+}
+```

LLMatic-main/centroids_100_2.dat

@@ -0,0 +1,100 @@
+0.05459372771370208 0.135647896224746 
+0.6227804995491316 0.620973950991632 
+0.21168382671045838 0.8904385735015196 
+0.700275078817918 0.04356929374920682 
+0.8080341614153019 0.8058439837924393 
+0.3922132877052243 0.44063549660620993 
+0.8132389948752509 0.28784541640638156 
+0.45653356129741574 0.17266145801433014 
+0.0355106346127958 0.5629750610104168 
+0.4962906743786589 0.9569775031930101 
+0.9468102033875045 0.5258146523863926 
+0.1475730605346059 0.7122914637978367 
+0.2520828797888778 0.23808718686363978 
+0.4889507652592643 0.7755743489096112 
+0.19174969726624036 0.4952428499828997 
+0.8906526372894228 0.04193851585183922 
+0.9329030540827177 0.7506930855472149 
+0.59136282962791 0.41329807515041905 
+0.6355953508715751 0.3148937220930016 
+0.34723257319648576 0.6313767569569555 
+0.8815166115208439 0.22332231114693418 
+0.16358808916536455 0.295756441443098 
+0.24582569523354886 0.04910604002517388 
+0.5311846180954723 0.047691255520401876 
+0.8835962151865034 0.3372294169283959 
+0.0558813473930706 0.32087168690965023 
+0.043100462234259895 0.7647606812046712 
+0.7992651087241485 0.6988371538857203 
+0.9335042331220507 0.958837206129763 
+0.7263123112928429 0.613156974524149 
+0.27566551680923307 0.9583726919066804 
+0.4113011442097963 0.8474048251095265 
+0.7648189300879635 0.9529300438351493 
+0.5204926958535333 0.8738166487640304 
+0.5173661044627706 0.605198351446326 
+0.6032346523608383 0.8109905514114759 
+0.15498025316141673 0.1795887949460116 
+0.8845255523855198 0.6558801579256681 
+0.04717355622805225 0.9565417381877419 
+0.6295421858505851 0.9477822599657907 
+0.049858623877461095 0.044047166614117006 
+0.36395118944143035 0.1422264461018352 
+0.5555212444348316 0.1386768859008406 
+0.6888516615376656 0.4103856698335078 
+0.358433507681592 0.245396284336028 
+0.08352166196689959 0.49747277473767343 
+0.8699091661840197 0.45887585887627474 
+0.30549892309247895 0.5158147550419125 
+0.767447776570032 0.20803603536067589 
+0.4916614464244339 0.40964981895759284 
+0.5649528510723348 0.7071586302130887 
+0.0489799468380579 0.6648339986768609 
+0.5678797326828701 0.5066914140894362 
+0.13053358223511236 0.6040455367857318 
+0.9523629983889952 0.8501259166437479 
+0.15043319239103564 0.06641673226779093 
+0.29576443400742447 0.7667357093261276 
+0.4560765848185523 0.2840236668786583 
+0.17850076319699248 0.39679824938101965 
+0.8051521183656705 0.040711810741070276 
+0.6180577706820564 0.05778578099655124 
+0.8337909366205374 0.5726154841341241 
+0.31219039063739523 0.8638317318810518 
+0.9573217294231177 0.16440921354138838 
+0.2335528921139179 0.5898913988893592 
+0.2890664948176134 0.3242451831765145 
+0.4088212620352536 0.5612304872490749 
+0.44236261674681854 0.0590579148436543 
+0.9561745236490129 0.40188271811886944 
+0.060514336770431054 0.23172999841610276 
+0.34429632198928545 0.044405342957909266 
+0.04862366994210715 0.8683195689963357 
+0.789472912718755 0.3895622749717449 
+0.14897018829209263 0.955949071967015 
+0.7025234749615219 0.8542659924064914 
+0.9635145859127145 0.636144022104827 
+0.6935794436896556 0.7316289472269284 
+0.12640758469836644 0.8311643467208365 
+0.37083764586429246 0.7416708401537713 
+0.681214410600326 0.1315322171723074 
+0.7636870490838004 0.4959350927934502 
+0.29447134875984765 0.41849115719065805 
+0.25263710923999294 0.6823220868523201 
+0.8528831212268677 0.8985654813563939 
+0.05794846541806997 0.41302290581054446 
+0.7200696766632078 0.29766992366433287 
+0.8679498439026898 0.12812108468225358 
+0.3840673709043919 0.9525642251303081 
+0.4783679209709438 0.5040654069908896 
+0.9635616955068795 0.06376826838796357 
+0.775946045727018 0.12293523384537769 
+0.393612762751659 0.3509341673137685 
+0.6621444086504229 0.5143661132500527 
+0.9674465096657459 0.28511831400310084 
+0.2084787407737561 0.791700190836919 
+0.5516163879003518 0.23245824407980603 
+0.2570247451101261 0.14089619040634005 
+0.45352888525950436 0.6759020257142933 
+0.5446463916179347 0.3331631711205768 
+0.651900557168317 0.20902488539835712 

LLMatic-main/codegen_tests.py

@@ -0,0 +1,306 @@
+from mutation_models import codegen_mutate
+from utils import read_python_file, extract_code_section, get_class
+from evaluations import is_trainable
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+import torchvision.transforms as transforms
+
+import numpy as np
+import os
+import random
+import hydra
+import ray
+
+from conf.config import Config
+
+
+class Net(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.conv1 = nn.Conv2d(3, 1, 1)
+        self.fc1 = nn.Linear(1024, 10)
+
+    def forward(self, x):
+        x = F.relu(self.conv1(x))
+        x = torch.flatten(x, 1)
+        x = F.relu(self.fc1(x))
+        return x
+
+
+def fixer_test(cfg):
+    ray.init()
+    exp_name = "0003"
+    path_nets = f"codegen_tests/{exp_name}"
+    if not os.path.exists(path_nets):
+        os.makedirs(os.path.normpath(path_nets))
+    main_net_focus = read_python_file(os.path.normpath("/mnt/lustre/users/mnasir/NAS-LLM/best_net.py"))
+    print(main_net_focus)
+    #fixing_prompt = '"""Create another class that inherits from nn.Module to edit the above network such that it works when torch.zeros((1, 3, 32, 32)) is passed as an input"""'
+    #fixing_prompt = '"""Fix the above neural network bu editing the layers such that it takes torch.zeros((1,3,32,32)) as inputs and output as tensor of size torch.size((1,10))"""'
+    fixing_prompt = '"""The above neural network does not work in the current form. Add or delete layers to fix the above neural network such that it takes torch.zeros((1,3,32,32)) as inputs and output as tensor of size torch.size((1,10))"""'
+    for i in range(10):
+        (print(f"Test {i}"))
+        
+        fixed_net_results = codegen_mutate.remote(cfg=cfg, prompt=main_net_focus + "\n" + fixing_prompt, temperature = 0.1)
+        extract_code_section(ray.get(fixed_net_results), fixing_prompt, file_path=os.path.normpath(f"{path_nets}/fixed_net_{i}.py"))
+        print(read_python_file(os.path.normpath(f"{path_nets}/fixed_net_{i}.py")))
+        try:
+            Net = get_class(os.path.normpath(f"{path_nets}/fixed_net_{i}.py"))             
+            net = Net()
+            if is_trainable(net):
+                print("TRUE")
+            else:
+                print("FALSE")
+        except Exception as e:
+            print(f"FALSE because of {e}")
+
+
+def init_net_test(cfg):
+    ray.init()
+    exp_name = "init_net_test_0007"
+    path_nets = f"codegen_tests/{exp_name}"
+    if not os.path.exists(path_nets):
+        os.makedirs(os.path.normpath(path_nets))
+    init_net_prompt = '"""Create a simple neural network class that inherits from nn.Module pytorch class. It should accept input tensor of size 32 x 32 x 3 and output 10 neurons for classification task"""'
+    for i in range(10):
+        (print(f"Test {i}"))
+        
+        fixed_net_results = codegen_mutate.remote(cfg=cfg, prompt=init_net_prompt, temperature = 0.3)
+        extract_code_section(ray.get(fixed_net_results), init_net_prompt, file_path=os.path.normpath(f"{path_nets}/init_net_{i}.py"))
+        print(read_python_file(os.path.normpath(f"{path_nets}/init_net_{i}.py")))
+        try:
+            Net = get_class(os.path.normpath(f"{path_nets}/init_net_{i}.py"))             
+            net = Net()
+            if is_trainable(net):
+                print("TRUE")
+            else:
+                print("FALSE")
+        except Exception as e:
+            print(f"FALSE because of {e}")
+
+def mutation_test(cfg):
+    ray.init()
+    exp_name = "mutation_test_0005"
+
+    prompts = ['"""Add a layer to improve the above network"""',
+               '"""Delete a layer to improve the above network"""',
+               '"""Increase the width of the above neural network"""',
+               '"""Decrease the width of the above neural network"""',
+               '"""Increase the depth of the above neural network"""',
+               '"""Decrease the depth of the above neural network"""',
+               '"""Add fully connected layer to improve the above network"""',
+               '"""Add convolutional layer to improve the above network"""',
+               '"""Add pooling layer to improve the above network"""',
+               '"""Add residual connection to improve the above network"""',
+               '"""Add multiple residual connections to improve the above network"""',
+               '"""Add dropout layer to improve the above network"""',
+               '"""Add normalization layer to improve the above network"""',
+                ]
+    path_nets = f"codegen_tests/{exp_name}"
+    if not os.path.exists(path_nets):
+        os.makedirs(os.path.normpath(path_nets))
+
+    
+    seed_value = 1
+    for prompt in prompts:
+        (print(f"Test on prompt: {prompt}"))
+
+        seed_value = seed_value + 10
+        torch.manual_seed(seed_value)
+
+        mutation_prompt = \
+        f'''class Net(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.conv1 = nn.Conv2d(3, 1, 1)
+        self.fc1 = nn.Linear(1024, 10)
+    def forward(self, x):
+        x = F.relu(self.conv1(x))
+        x = torch.flatten(x, 1)
+        x = F.relu(self.fc1(x))
+        return x
+    
+{prompt}'''
+        
+        fixed_net_results = codegen_mutate.remote(cfg=cfg, prompt=mutation_prompt, temperature = 0.6)
+        extract_code_section(ray.get(fixed_net_results), mutation_prompt, file_path=os.path.normpath(f"{path_nets}/mu_net_{prompt}.py"))
+        print(read_python_file(os.path.normpath(f"{path_nets}/mu_net_{prompt}.py")))
+        try:
+            Net = get_class(os.path.normpath(f"{path_nets}/mu_net_{prompt}.py"))             
+            net = Net()
+            if is_trainable(net):
+                print("TRUE")
+            else:
+                print("FALSE")
+        except Exception as e:
+            print(f"FALSE because of {e}")
+
+
+def crossover_test(cfg):
+    ray.init()
+    exp_name = "crossover_test_0001"
+
+    prompt = '''
+
+    class Net(nn.Module):
+        def __init__(self):
+            super(Net, self).__init__()
+            self.conv1 = nn.Conv2d(3, 6, 5)
+            self.pool = nn.MaxPool2d(2, 2)
+            self.conv2 = nn.Conv2d(6, 16, 5)
+            self.fc1 = nn.Linear(16 * 5 * 5, 120)
+            self.fc2 = nn.Linear(120, 84)
+            self.fc3 = nn.Linear(84, 10)
+
+        def forward(self, x):
+            x = self.pool(F.relu(self.conv1(x)))
+            x = self.pool(F.relu(self.conv2(x)))
+            x = x.view(-1, 16 * 5 * 5)
+            x = F.relu(self.fc1(x))
+            x = F.relu(self.fc2(x))
+            x = self.fc3(x)
+            return x
+
+    class Net2(nn.Module):
+        def __init__(self):
+            super(Net2, self).__init__()
+            self.conv1 = nn.Conv2d(3, 1, 1)
+            self.fc1 = nn.Linear(1024 128)
+            self.fc2 = nn.Linear(128, 64)
+            self.fc3 = nn.Linear(64, 32)
+            self.fc4 = nn.Linear(32, 10)
+    
+        def forward(self, x):
+            x = self.pool(F.relu(self.conv1(x)))
+            x = torch.flatten(x, 1)
+            x = F.relu(self.fc1(x))
+            x = F.relu(self.fc2(x))
+            x = F.relu(self.fc3(x))
+            x = self.fc4(x)
+            return x
+    
+    """Perform a crossover between above two neural networks and create a third neural network class that gets the best layers from above two networks"""'''
+    path_nets = f"codegen_tests/{exp_name}"
+    if not os.path.exists(path_nets):
+        os.makedirs(os.path.normpath(path_nets))
+    for i in range(10):
+        (print(f"Test {i}"))
+        
+        fixed_net_results = codegen_mutate.remote(cfg=cfg, prompt=prompt, temperature = 0.6)
+        extract_code_section(ray.get(fixed_net_results), prompt, file_path=os.path.normpath(f"{path_nets}/init_net_{i}.py"))
+        print(read_python_file(os.path.normpath(f"{path_nets}/init_net_{i}.py")))
+        try:
+            Net = get_class(os.path.normpath(f"{path_nets}/init_net_{i}.py"))             
+            net = Net()
+            if is_trainable(net):
+                print("TRUE")
+            else:
+                print("FALSE")
+        except Exception as e:
+            print(f"FALSE because of {e}")
+
+
+def diff_mutate_test(cfg):
+    ray.init()
+    exp_name = "diff_mutate_test_0004"
+    path_nets = f"codegen_tests/{exp_name}"
+    if not os.path.exists(path_nets):
+        os.makedirs(os.path.normpath(path_nets))
+    diff_prompts = ['<NME> initial_net.py\n'
+    '<BFE> import torch\n'
+    'import torch.nn as nn\n'
+    'import torch.nn.functional as F\n'
+    '"""Returns a pytorch neural network class that takes an image of 3 x 32 x 32 as input and outputs 10 neurons."""\n'
+    'class Net(nn.Module):\n'
+    '   def __init__(self):\n'
+    '       super().__init__()\n'
+    '       self.conv1 = nn.Conv2d(3, 1, 1)\n'
+    '       self.fc1 = nn.Linear(1024, 10)\n'
+    '   def forward(self, x):\n'
+    '       x = F.relu(self.conv1(x))\n'
+    '       x = torch.flatten(x, 1)\n'
+    '       x = F.relu(self.fc1(x))\n'
+    '       return x\n'
+    '<MSG> Added a nn.Conv2d layer to improve the neural network.\n',
+    '<NME> initial_net.py\n'
+    '<BFE> import torch\n'
+    'import torch.nn as nn\n'
+    'import torch.nn.functional as F\n'
+    '"""Returns a pytorch neural network class that takes an image of 3 x 32 x 32 as input and outputs 10 neurons."""\n'
+    'class Net(nn.Module):\n'
+    '   def __init__(self):\n'
+    '       super().__init__()\n'
+    '       self.conv1 = nn.Conv2d(3, 1, 1)\n'
+    '       self.fc1 = nn.Linear(1024, 10)\n'
+    '   def forward(self, x):\n'
+    '       x = F.relu(self.conv1(x))\n'
+    '       x = torch.flatten(x, 1)\n'
+    '       x = F.relu(self.fc1(x))\n'
+    '       return x\n'
+    '<MSG> Added a nn.Linear layer to improve the neural network.\n',
+    '<NME> initial_net.py\n'
+    '<BFE> import torch\n'
+    'import torch.nn as nn\n'
+    'import torch.nn.functional as F\n'
+    '"""Returns a pytorch neural network class that takes an image of 3 x 32 x 32 as input and outputs 10 neurons."""\n'
+    'class Net(nn.Module):\n'
+    '   def __init__(self):\n'
+    '       super().__init__()\n'
+    '       self.conv1 = nn.Conv2d(3, 1, 1)\n'
+    '       self.fc1 = nn.Linear(1024, 10)\n'
+    '   def forward(self, x):\n'
+    '       x = F.relu(self.conv1(x))\n'
+    '       x = torch.flatten(x, 1)\n'
+    '       x = F.relu(self.fc1(x))\n'
+    '       return x\n'
+    '<MSG> Added a nn.Conv2d and a nn.Linear layer to improve the neural network.\n',
+    '<NME> initial_net.py\n'
+    '<BFE> import torch\n'
+    'import torch.nn as nn\n'
+    'import torch.nn.functional as F\n'
+    '"""Returns a pytorch neural network class that takes an image of 3 x 32 x 32 as input and outputs 10 neurons."""\n'
+    'class Net(nn.Module):\n'
+    '   def __init__(self):\n'
+    '       super().__init__()\n'
+    '       self.conv1 = nn.Conv2d(3, 1, 1)\n'
+    '       self.fc1 = nn.Linear(1024, 10)\n'
+    '   def forward(self, x):\n'
+    '       x = F.relu(self.conv1(x))\n'
+    '       x = torch.flatten(x, 1)\n'
+    '       x = F.relu(self.fc1(x))\n'
+    '       return x\n'
+    '<MSG> Added layers to improve the neural network.\n']
+    for i,diff_prompt in enumerate(diff_prompts):
+        (print(f"Test {i}"))
+        
+        fixed_net_results = codegen_mutate.remote(cfg=cfg, prompt=diff_prompt, temperature = 0.8)
+        res = ray.get(fixed_net_results)
+        print(res)
+        extract_code_section(res, diff_prompt, file_path=os.path.normpath(f"{path_nets}/diff_prompt_{i}.py"))
+        print(read_python_file(os.path.normpath(f"{path_nets}/diff_prompt_{i}.py")))
+        try:
+            Net = get_class(os.path.normpath(f"{path_nets}/diff_prompt_{i}.py"))             
+            net = Net()
+            if is_trainable(net):
+                print("TRUE")
+            else:
+                print("FALSE")
+        except Exception as e:
+            print(f"FALSE because of {e}")
+
+
+@hydra.main(version_base="1.3.0", config_path="conf", config_name="config")    
+def main(cfg: Config):
+    #fixer_test(cfg)
+    #init_net_test(cfg)
+    #mutation_test(cfg)
+    crossover_test(cfg)
+    #diff_mutate_test(cfg)
+
+
+if __name__ == "__main__":
+    main()

LLMatic-main/conf/config.py

@@ -0,0 +1,43 @@
+from dataclasses import MISSING, dataclass
+import typing
+import multiprocessing as mp
+
+from hydra.core.config_store import ConfigStore
+from map_elites import common as cm
+
+@dataclass
+class Config:
+    """This class provides a "schema" for the config file, validating types."""
+
+    RANDOM_INIT_NETS: int = 10 # INIT Nets to be accepted in archive
+    INIT_NUM_NETS: int = 10 # INIT Nets created per generation
+    ROLL_OUTS: int = 4 # For GPU training. ROLL_OUTS * (INIT_NUM_NETS or NUM_NETS) = Total nets created in each generation
+    NUM_NETS: int = 10 # Mutation and crossover nets to be created
+    START_FROM_CHECKPOINT: bool = False
+    RANDOM_NETWORKS: bool = False
+
+    MUTATION: str = "codegen-6B-mono"
+    #MUTATION: str = "codex"
+
+    GENERATIONS: int = 200
+    NET_TRAINING_EPOCHS: int = 50
+    TEMPERATURE: float = 0.0000000
+
+    DEVICE: str = "cuda" # options: ["cuda", "cpu", "both"]
+
+    NUM_PROCESSES: int = mp.cpu_count() - 1
+    RAY: bool = True
+    NUM_CPUS: int = NUM_PROCESSES
+    NUM_GPUS: int = 1 # per task
+    NUM_GPUS_TOTAL: int  = 4 # total available
+ 
+    # MAP-ELITES
+
+    DIM_MAP: int = 2
+    N_NICHES: int = 100
+    
+    SAVE_DIR: str = "./"
+             
+
+cs = ConfigStore.instance()
+cs.store(name="base_config", node=Config)

LLMatic-main/conf/config.yaml

@@ -0,0 +1,23 @@
+defaults:
+  - base_config
+  - _self_
+
+  - override hydra/launcher: submitit_slurm 
+  # - override hydra/launcher: submitit_local
+
+
+hydra:
+  sweeper:
+    params:
+      NUM_NETS: 10, 20
+      # GENERATIONS: 25, 50, 100
+      NET_TRAINING_EPOCHS: 10, 30
+      TEMPERATURE: 0.3, 0.5, 0.7
+
+  # SLURM-specific arguments
+  launcher:
+    tasks_per_node: 1
+    cpus_per_task: 1
+    gpus_per_node: 1
+    timeout_min: 2880
+    mem_gb: 30

LLMatic-main/datasets.py

@@ -0,0 +1,29 @@
+import torch
+import torchvision
+import torchvision.transforms as transforms
+
+
+import numpy as np
+
+
+def get_datasets():
+    
+    transform = transforms.Compose(
+        [transforms.ToTensor(),
+         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
+
+    batch_size = 1
+
+    trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
+                                            download=True, transform=transform)
+    trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,
+                                              shuffle=True, num_workers=1)
+
+    validset = torchvision.datasets.CIFAR10(root='./data', train=False,
+                                           download=True, transform=transform)
+    validloader = torch.utils.data.DataLoader(validset, batch_size=batch_size,
+                                             shuffle=False, num_workers=1)
+
+    return trainset, trainloader, validset, validloader
+
+

LLMatic-main/environment.yaml

@@ -0,0 +1,91 @@
+name: llmatic
+channels:
+  - defaults
+dependencies:
+  - ca-certificates=2023.01.10=haa95532_0
+  - certifi=2022.12.7=py39haa95532_0
+  - openssl=1.1.1t=h2bbff1b_0
+  - pip=22.3.1=py39haa95532_0
+  - python=3.9.16=h6244533_0
+  - setuptools=65.6.3=py39haa95532_0
+  - sqlite=3.40.1=h2bbff1b_0
+  - tzdata=2022g=h04d1e81_0
+  - vc=14.2=h21ff451_1
+  - vs2015_runtime=14.27.29016=h5e58377_2
+  - wheel=0.38.4=py39haa95532_0
+  - wincertstore=0.2=py39haa95532_2
+  - pip:
+    - aiohttp==3.8.4
+    - aiosignal==1.3.1
+    - antlr4-python3-runtime==4.9.3
+    - async-timeout==4.0.2
+    - attrs==22.2.0
+    - charset-normalizer==3.0.1
+    - click==8.1.3
+    - cloudpickle==2.2.1
+    - colorama==0.4.6
+    - contourpy==1.0.7
+    - cycler==0.11.0
+    - filelock==3.9.0
+    - flopth==0.1.3
+    - fonttools==4.38.0
+    - frozenlist==1.3.3
+    - fvcore==0.1.5.post20221221
+    - huggingface-hub==0.12.1
+    - hydra-core==1.3.1
+    - hydra-submitit-launcher==1.2.0
+    - idna==3.4
+    - importlib-resources==5.12.0
+    - inquirerpy==0.3.4
+    - iopath==0.1.10
+    - joblib==1.2.0
+    - jsonschema==4.21.1
+    - jsonschema-specifications==2023.12.1
+    - kiwisolver==1.4.4
+    - levenshtein==0.20.9
+    - matplotlib==3.7.0
+    - msgpack==1.0.8
+    - multidict==6.0.4
+    - networkx==3.0
+    - numpy==1.24.2
+    - omegaconf==2.3.0
+    - openai==0.26.5
+    - packaging==23.0
+    - pandas==1.5.3
+    - pfzy==0.3.4
+    - pillow==9.4.0
+    - portalocker==2.7.0
+    - prompt-toolkit==3.0.37
+    - protobuf==5.26.1
+    - pyparsing==3.0.9
+    - python-dateutil==2.8.2
+    - pytz==2022.7.1
+    - pywin32==305
+    - pyyaml==6.0
+    - rapidfuzz==2.13.7
+    - ray==2.10.0
+    - referencing==0.34.0
+    - regex==2022.10.31
+    - requests==2.28.2
+    - rpds-py==0.18.0
+    - sacremoses==0.0.53
+    - scikit-learn==1.2.2
+    - scipy==1.10.1
+    - six==1.16.0
+    - submitit==1.4.5
+    - tabulate==0.9.0
+    - termcolor==2.2.0
+    - threadpoolctl==3.1.0
+    - tokenizers==0.13.2
+    - torch==1.13.1
+    - torchvision==0.14.1
+    - tqdm==4.64.1
+    - transformers==4.25.1
+    - typing-extensions==4.5.0
+    - urllib3==1.26.14
+    - wcwidth==0.2.6
+    - xmltodict==0.13.0
+    - yacs==0.1.8
+    - yarl==1.8.2
+    - zipp==3.14.0
+prefix: C:\Users\DELL\anaconda3\envs\llmatic

LLMatic-main/evaluations.py

@@ -0,0 +1,41 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class Net(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.conv1 = nn.Conv2d(3, 1, 1, bias=False)
+        self.conv2 = nn.Conv2d(3, 1, 1, bias=False)
+        self.fc1 = nn.Linear(1024, 10)
+
+    def forward(self, x):
+        x = F.relu(self.conv1(x))
+        x = F.relu(self.conv2(x))
+        x = torch.flatten(x, 1)
+        x = F.relu(self.fc1(x))
+        return x
+
+
+
+net = Net()
+def is_trainable(net):
+    zeros = torch.zeros((1, 3, 32, 32))  # input tensor of shape (batch_size, channels, height, width)
+
+    # Check that we can pass a dummy input through the network without errors.
+    try:
+        output = net(zeros)
+    except Exception as e:
+        return False
+    
+    # Network output shape must match number of classes in CIFAR-10.
+    if output.shape != (1, 10):
+        return False
+
+    return True
+
+def main():
+    print(is_trainable(net))
+
+if __name__ == '__main__':
+    main()

LLMatic-main/initial_net.py

@@ -0,0 +1,13 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+class Net(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.conv1 = nn.Conv2d(3, 1, 1)
+        self.fc1 = nn.Linear(1024, 10)
+    def forward(self, x):
+        x = F.relu(self.conv1(x))
+        x = torch.flatten(x, 1)
+        x = F.relu(self.fc1(x))
+        return x

LLMatic-main/llmatic.py

@@ -0,0 +1,566 @@
+#| This file has a major part from the pymap_elites framework.
+#| Copyright 2019, INRIA
+#| Main contributor(s):
+#| Jean-Baptiste Mouret, jean-baptiste.mouret@inria.fr
+#| Eloise Dalin , eloise.dalin@inria.fr
+#| Pierre Desreumaux , pierre.desreumaux@inria.fr
+#|
+#|
+#| **Main paper**: Mouret JB, Clune J. Illuminating search spaces by
+#| mapping elites. arXiv preprint arXiv:1504.04909. 2015 Apr 20.
+#|
+#| This software is governed by the CeCILL license under French law
+#| and abiding by the rules of distribution of free software.  You
+#| can use, modify and/ or redistribute the software under the terms
+#| of the CeCILL license as circulated by CEA, CNRS and INRIA at the
+#| following URL "http://www.cecill.info".
+#|
+#| As a counterpart to the access to the source code and rights to
+#| copy, modify and redistribute granted by the license, users are
+#| provided only with a limited warranty and the software's author,
+#| the holder of the economic rights, and the successive licensors
+#| have only limited liability.
+#|
+#| In this respect, the user's attention is drawn to the risks
+#| associated with loading, using, modifying and/or developing or
+#| reproducing the software by the user in light of its specific
+#| status of free software, that may mean that it is complicated to
+#| manipulate, and that also therefore means that it is reserved for
+#| developers and experienced professionals having in-depth computer
+#| knowledge. Users are therefore encouraged to load and test the
+#| software's suitability as regards their requirements in conditions
+#| enabling the security of their systems and/or data to be ensured
+#| and, more generally, to use and operate it in the same conditions
+#| as regards security.
+#|
+#| The fact that you are presently reading this means that you have
+#| had knowledge of the CeCILL license and that you accept its terms.
+
+import math
+import numpy as np
+import multiprocessing
+
+# from scipy.spatial import cKDTree : TODO -- faster?
+from sklearn.neighbors import KDTree
+
+from map_elites import common as cm
+
+from flopth import flopth
+
+from datasets import get_datasets
+from utils import read_python_file, extract_code, get_net_name, extract_code_section, get_class, csv_writer, get_max_curiosity, get_max_fitness, count_unique_components, count_parameters, get_network_width_depth_ratio
+from utils import Net
+from mutation_models import codegen_mutate, codex_mutate, replace_word_mutation
+from evaluations import is_trainable
+from train import train_net_on_cpu, train_net_on_gpu, forward_pass_on_cpu,forward_pass_on_gpu, detect_layers, transfer_weights
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+import torchvision.transforms as transforms
+
+import numpy as np
+import os
+import random
+import hydra
+import ray
+from fvcore.nn import FlopCountAnalysis
+
+from conf.config import Config
+
+
+
+
+def __add_to_archive(s, centroid, archive, kdt, type_ind):
+    niche_index = kdt.query([centroid], k=1)[1][0][0]
+    niche = kdt.data[niche_index]
+    n = cm.make_hashable(niche)
+    s.centroid = n
+    if n in archive:
+        if type_ind == "network": 
+            if s.fitness < archive[n].fitness:
+                archive[n] = s
+                return True
+        elif type_ind == "prompt":
+            if s.fitness > archive[n].fitness:
+                archive[n] = s
+                return True
+        return False
+    else:
+        archive[n] = s
+        return True
+
+
+
+def to_specie(net,fit,desc,net_path):
+    return cm.Species(net, desc, fit, net_path)
+
+# map-elites algorithm (CVT variant)
+
+@hydra.main(version_base="1.3.0", config_path="conf", config_name="config")    
+def main(cfg: Config):
+    """CVT MAP-Elites
+       Vassiliades V, Chatzilygeroudis K, Mouret JB. Using centroidal voronoi tessellations to scale up the multidimensional archive of phenotypic elites algorithm. IEEE Transactions on Evolutionary Computation. 2017 Aug 3;22(4):623-30.
+
+       Format of the logfile: evals archive_size max mean median 5%_percentile, 95%_percentile
+
+    """
+    if cfg.DEVICE == "cuda":
+        ray.init(num_gpus=cfg.NUM_GPUS_TOTAL) 
+    else:
+        ray.init()
+    mutation_fn = codex_mutate if cfg.MUTATION == "codex" else codegen_mutate
+
+    prompts = ['"""Add a layer to improve the above network"""',
+               '"""Delete a layer to improve the above network"""',
+               '"""improve the above network"""',
+               '"""improve the above network by reducing the size drastically"""',
+               '"""improve the above network by increasing the size drastically"""',
+               # Specific prompts below
+               '"""Add fully connected layer to improve the above network"""',
+               '"""Add convolutional layer to improve the above network"""',
+               '"""Add pooling layer to improve the above network"""',
+               '"""Add residual connection to improve the above network"""',
+               '"""Add multiple residual connections to improve the above network"""',
+               '"""Add dropout layer to improve the above network"""',
+               '"""Add normalization layer to improve the above network"""',
+               '"""Add recurrent layer to improve the above network"""',
+                ]
+    probabilities = [1.0 / len(prompts)] * len(prompts)
+
+    prompt_score = {}
+    for prompt in prompts:
+        prompt_score[prompt] = 0
+    prompt_score["None"] = 0
+
+    prompt_to_int = {}
+    int_to_prompt = {}
+    for i,prompt in enumerate(prompts):
+        prompt_to_int[prompt] = i
+        int_to_prompt[i] = prompt
+    
+    if cfg.START_FROM_CHECKPOINT:
+        best_net = read_python_file("initial_net.py")
+        init_gen = 4
+        print(f"Starting from the following network:\n{best_net}")
+    else:
+        init_net = read_python_file(os.path.normpath(f"{cfg.SAVE_DIR}/initial_net.py"))
+        init_gen = 0
+        print(f"Starting from the following network:\n{init_net}")
+    
+    
+    # create the CVT
+    params = cm.default_params
+    c = cm.cvt(cfg.N_NICHES,cfg.DIM_MAP,
+              params['cvt_samples'], params['cvt_use_cache'])
+    kdt = KDTree(c, leaf_size=30, metric='euclidean')
+    cm.__write_centroids(c)
+
+    prompt_archive = {} # init archive (empty)
+    nets_archive = {}
+    n_evals = 0 # number of evaluations since the beginning
+    b_evals = 0 # number evaluation since the last dump
+
+    curios_net_path = os.path.normpath(f"{cfg.SAVE_DIR}/initial_net.py")
+
+    c_net_str = read_python_file(curios_net_path)
+    Net = get_class(curios_net_path)
+    curios_net = Net()
+    curios_prompt = prompts[0]
+    # main loop
+
+    temperature_start = 0.6
+    prev_best_score = np.inf
+    prev_best_net_path = None
+    prev_best_prompt = None
+
+    exp_name = f"gen-nets_{cfg.MUTATION}_networks-{cfg.NUM_NETS}_temp-{cfg.TEMPERATURE}_net-training-epochs-{cfg.NET_TRAINING_EPOCHS}_niches-{cfg.N_NICHES}_infer-and-flops-as-bd"
+    path_nets = f"{cfg.SAVE_DIR}/logs/{exp_name}"
+    os.makedirs(path_nets,exist_ok=True)
+    log_file = open(os.path.normpath(f"{path_nets}/cvt.dat"),"w")
+    out_file = os.path.normpath(f"{path_nets}/exp_results.csv")        
+    csv_writer(["generations", "best_loss", "used_prompt"],out_file)
+            
+
+    for gen_i in range(init_gen, cfg.GENERATIONS):
+
+        generated_nets = []
+        
+        # random initialization
+        (print(f"[Generation: {gen_i}]"))
+
+        if (len(nets_archive.keys()) < cfg.RANDOM_INIT_NETS) and (len(prompt_archive.keys()) < (int(cfg.RANDOM_INIT_NETS*2))):#params['random_init'] * n_niches:
+            
+            
+            for _i in range(cfg.ROLL_OUTS):
+                selected_prompts = []
+                generated_net_results = []
+                selected_prompts = random.choices(prompts, weights=probabilities,k=cfg.INIT_NUM_NETS)
+                for i in range(0, cfg.INIT_NUM_NETS):
+                    print(f"Selected prompt for generations: {selected_prompts[i]}")
+
+                for prompt in selected_prompts:
+                    generated_net_results.append(mutation_fn.remote(cfg=cfg, prompt=init_net + "\n" + prompt, temperature=temperature_start))
+
+
+                for i,generated_net in enumerate(generated_net_results):
+                    generated_nets.append((ray.get(generated_net),selected_prompts[i],temperature_start))
+                
+        else:  # variation/selection loop
+            
+            evo_operator = random.choices(["mutation","crossover"], weights=[0.85,0.15])[0]
+            
+            print(f"Performing {evo_operator}")
+            for _i in range(cfg.ROLL_OUTS):
+                generated_net_results = []
+                if evo_operator == "mutation":
+            
+                    if len(nets_archive.keys()) < 3:
+                        n_nets = 1
+                        selection = 0
+                    else:
+                        n_nets = 3
+                        selection = random.randint(0, 2)
+
+                    curios_nets = []
+                    curios_prompts = []
+                    curios_temps = []
+                    curios_net_paths = []
+
+                    n_curios_nets = get_max_fitness(nets_archive.values(),n_nets)
+                    for n in n_curios_nets:
+                        curios_nets.append(n.x)
+                        curios_net_paths.append(n.net_path)
+
+                    n_prompts_temps = get_max_curiosity(prompt_archive.values(),n_nets)
+                    for pt in n_prompts_temps:
+                        curios_prompts.append(pt.desc[0])
+                        curios_temps.append(pt.desc[1])
+
+                    curios_net = curios_nets[selection]
+                    curios_temp = curios_temps[selection]
+                    curios_net_path = curios_net_paths[selection]
+
+                    curios_temp_ray = []
+                    curios_prompt_ray = []
+
+
+                    for i in range(cfg.NUM_NETS):
+                        # Temperature mutation
+
+                        if i == 1:
+                            curios_temp = curios_temps[selection]
+                        else:
+                            curios_temp += random.uniform(-0.1,0.1)
+                            if curios_temp > 1.0:
+                                curios_temp = 1.0
+                            elif curios_temp < 0.1:
+                                curios_temp = 0.1
+
+                        curios_prompt = curios_prompts[selection]
+
+                        curios_temp_ray.append(curios_temp)
+                        curios_prompt_ray.append(curios_prompt)
+
+                        print(f"prompt in mutation is {curios_prompt}")
+
+                    for i in range(cfg.NUM_NETS):
+                        generated_net_results.append(mutation_fn.remote(cfg=cfg, prompt=read_python_file(curios_net_path) + "\n" + int_to_prompt[int(curios_prompt_ray[i])], temperature = curios_temp_ray[i]))
+
+                    for i,generated_net in enumerate(generated_net_results):
+                        generated_nets.append((ray.get(generated_net),int_to_prompt[int(curios_prompt_ray[i])],curios_temp_ray[i]))
+
+                elif evo_operator == "crossover":
+
+                    if len(nets_archive.keys()) < 1:
+                        print("No crossover performed")
+                    else:
+                        curios_nets_ray = []
+                        curios_temps_ray = []
+
+                        curios_nets = []
+                        curios_prompts = []
+                        curios_temps = []
+                        curios_net_paths = []
+
+
+                        if len(nets_archive.keys()) < 2:
+                            selection = 0
+                            n_nets = 1
+                        else:
+                            selection = random.randint(1, len(nets_archive.keys())-1)# change it to 1
+                            n_nets = 2
+
+                        n_curios_nets = get_max_fitness(nets_archive.values(),n_nets)
+                        for n in n_curios_nets:
+                            curios_nets.append(n.x)
+                            curios_net_paths.append(n.net_path)
+
+                        n_prompts_temps = get_max_curiosity(prompt_archive.values(),n_nets)
+                        for pt in n_prompts_temps:
+                            curios_prompts.append(pt.desc[0])
+                            curios_temps.append(pt.desc[1])
+
+
+
+
+                        for i in range(0, cfg.NUM_NETS):
+
+                            curios_net_str = read_python_file(curios_net_paths[0])
+                            curios_net = curios_nets[0]
+                            
+                            curios_temp = curios_temps[0]
+                            curios_prompt = curios_prompts[0]
+
+
+                            curios_nets_ray.append(read_python_file(curios_net_paths[selection]))
+
+
+                        crossover_prompt = '"""Combine the above two neural networks and create a third neural network class that also inherits from nn.Module"""'
+
+                        for curios_2nd_net in curios_nets_ray:
+                            generated_net_results.append(mutation_fn.remote(cfg=cfg, prompt=curios_net_str + "\n" + curios_2nd_net + "\n" + crossover_prompt, temperature = curios_temp))
+
+                        for i,generated_net in enumerate(generated_net_results):
+                            generated_nets.append((ray.get(generated_net),int_to_prompt[int(curios_prompt)],curios_temp))
+
+ 
+                           
+        net_class_name = None
+        net_paths=[]
+        training_prompts=[]
+        training_nets=[]
+
+        for i, k in enumerate(generated_nets):
+            
+            invalid_net = False
+            generation, prompt, temperature = k
+
+            
+            net_path = os.path.normpath(f"{path_nets}/network_{gen_i}_{i}_{prompt[3:-3]}_{temperature}.py")
+            
+
+            if cfg.MUTATION == "codex":
+                    extract_code_section(generation['choices'][0]["text"], prompt, file_path=net_path)
+            elif cfg.MUTATION.startswith("codegen"):
+                extract_code_section(generation, prompt, file_path=net_path)  
+            main_net_focus = read_python_file(net_path)
+            print(f"Net in focus:\n {main_net_focus}")
+
+            if not invalid_net:
+                try:
+                    Net = get_class(net_path)             
+                    net = Net()
+                except Exception as e:
+                    net = curios_net
+                    net_path = curios_net_path
+                    if isinstance(curios_prompt,str):
+                        prompt = curios_prompt
+                    elif isinstance(curios_prompt,float) or isinstance(curios_prompt,int):
+                        prompt = int_to_prompt[int(curios_prompt)]
+                    
+                #breakpoint()
+                try:
+                    is_t = is_trainable(net)
+                except Exception:
+                    is_t = False
+                if is_t:
+                    net_paths.append(net_path)
+                    training_prompts.append(prompt)
+                    training_nets.append(net)
+                else:
+                    invalid_net = True
+
+                    
+
+            if invalid_net:
+                print(f"The network at {net_path} is not trainable")
+        
+
+        if training_nets == []:
+            continue
+    
+        if (len(nets_archive.keys()) < cfg.RANDOM_INIT_NETS) and (len(prompt_archive.keys()) < (int(cfg.RANDOM_INIT_NETS*2))):
+
+            inter_results = []
+            losses = []
+            layers_c_net = detect_layers(curios_net)
+            if cfg.DEVICE == "cpu":
+                
+                for net in training_nets:
+                    for layer in layers_c_net:
+                        try:
+                            net = transfer_weights(curios_net,net,layer)
+                            print(f"Weights transferred successfully")
+                        except Exception:
+                            print("Weights can not transfer")
+                for net in training_nets:
+                    inter_results.append(forward_pass_on_gpu.remote(net))
+            elif cfg.DEVICE == "cuda":
+                    for net in training_nets:
+                        for layer in layers_c_net:
+                            try:
+                                net = transfer_weights(curios_net,net,layer)
+                                print(f"Weights transferred successfully")
+                            except Exception:
+                                print("Weights can not transfer")
+                    for net in training_nets:
+                        inter_results.append(forward_pass_on_gpu.remote(net))
+            else:
+                raise ValueError(f"{cfg.DEVICE} is not a valid device")
+            fitness = []
+            for i,result in enumerate(inter_results):
+                try:
+                    res = ray.get(result)
+                    fitness.append([res[0],training_prompts[i],temperature,net_paths[i],res[1]])
+
+                    if fitness[i][0] <= prev_best_score:
+                        prev_best_net_path = net_paths[i]
+                        prev_best_prompt = training_prompts[i]
+                        prev_best_score = fitness[i][0]
+                        temperature += 0.05
+
+                    else:
+                        temperature -= 0.05
+                    if temperature > 1.0:
+                        temperature = 1.0
+                    elif temperature < 0.1:
+                        temperature = 0.1
+
+                        fitness[i][2] = temperature
+                except Exception:
+                    print("not trainable due to fitness 1")
+        
+        else:
+
+            if cfg.RAY:
+                inter_results = []
+                losses = []
+                layers_c_net = detect_layers(curios_net)
+                if cfg.DEVICE == "cpu":
+                    for net in training_nets:
+                        for layer in layers_c_net:
+                            try:
+                                net = transfer_weights(curios_net,net,layer)
+                                print(f"Weights transferred successfully")
+                            except Exception:
+                                print("Weights can not transfer")
+                    for net in training_nets:
+                        inter_results.append(train_net_on_cpu.remote(net,cfg.NET_TRAINING_EPOCHS))
+
+                elif cfg.DEVICE == "both":
+                    for i, net in enumerate(training_nets):
+                        if i % 2 == 0:
+                            inter_results.append(train_net_on_cpu.remote(net,cfg.NET_TRAINING_EPOCHS))
+                        else:
+                            inter_results.append(train_net_on_gpu.remote(net,cfg.NET_TRAINING_EPOCHS))
+
+                elif cfg.DEVICE == "cuda":
+                    for net in training_nets:
+                        for layer in layers_c_net:
+                            try:
+                                net = transfer_weights(curios_net,net,layer)
+                                print(f"Weights transferred successfully")
+                            except Exception:
+                                print("Weights can not transfer")
+                    for net in training_nets:
+                        inter_results.append(train_net_on_gpu.remote(net,cfg.NET_TRAINING_EPOCHS))
+
+                else:
+                    raise ValueError(f"{cfg.DEVICE} is not a valid device")
+                fitness = []
+                for i,result in enumerate(inter_results):
+                    try:
+                        fitness.append([ray.get(result),training_prompts[i],temperature,net_paths[i],0.0])
+
+                        if fitness[i][0] <= prev_best_score:
+                            prev_best_net_path = net_paths[i]
+                            prev_best_prompt = training_prompts[i]
+                            prev_best_score = fitness[i][0]
+                            temperature += 0.05
+                        else:
+                            temperature -= 0.05
+
+                        if temperature > 1.0:
+                            temperature = 1.0
+                        elif temperature < 0.1:
+                            temperature = 0.1
+
+                        fitness[i][2] = temperature
+                    except Exception:
+                        print("not trainable due to fitness 2")
+                
+                try:
+                    infer_results = []
+                    for net in training_nets:
+                        infer_results.append(forward_pass_on_gpu.remote(net))
+                    
+                    
+                    for i,result in enumerate(infer_results):
+                        try:
+                            res = ray.get(result)
+                            fitness[i][4] = res[1]
+                        except Exception:
+                            print("not trainable due to inference speed 1")
+
+                    
+                except Exception:
+                    print("not trainable due to inference speed 2")
+                        
+        dummy_inputs = torch.zeros((1, 3, 32, 32))
+        net_beh_list = []
+        prompt_beh_list = []
+        for loss_x,prompt_x,temp_x,net_p,infer_speed in fitness:
+            _nn = read_python_file(net_p) #Avoiding sytax errors
+            Net = get_class(net_p)             
+            net = Net()
+            try:
+                flps = FlopCountAnalysis(net, dummy_inputs)
+                flops = flps.total()
+                depth_width_ratio = get_network_width_depth_ratio(net)
+
+            except Exception:
+                try:
+                    flps = FlopCountAnalysis(curios_net, dummy_inputs)
+                    flops = flps.total()
+                    depth_width_ratio = get_network_width_depth_ratio(curios_net)
+                except Exception:
+                    flops = 0
+                    depth_width_ratio = 1
+
+            print(f"flops, infer_speed, depth_width_ratio:  {flops}, {infer_speed}, {depth_width_ratio}")
+            s_net = to_specie(net,loss_x,np.array([depth_width_ratio, flops]),net_p)
+            net_added = __add_to_archive(s_net, s_net.desc, nets_archive, kdt, type_ind = "network")
+
+            if net_added:
+                prompt_fit = 1.0 
+            else:
+                prompt_fit = 0.0
+
+            s_prompt = to_specie(net,prompt_fit,np.array([prompt_to_int[prompt_x], temp_x]),net_p)
+            prompt_added = __add_to_archive(s_prompt, s_prompt.desc, prompt_archive, kdt, type_ind = "prompt")
+            if not net_added:
+                s_prompt.curiosity = s_prompt.curiosity - 0.5
+            elif net_added:
+                s_prompt.curiosity = s_prompt.curiosity + 1.0 
+        
+        if gen_i % 1 == 0:
+            
+            cm.__save_archive(nets_archive, gen_i,name="net")
+            cm.__save_archive(prompt_archive, gen_i,name="prompt")
+        # write log
+        if log_file != None:
+            fit_list = np.array([x.fitness for x in nets_archive.values()])
+            log_file.write("{} {} {} {} {} {} {}\n".format(gen_i, len(nets_archive.keys()), len(prompt_archive.keys()),
+                    fit_list.min(), np.mean(fit_list), np.median(fit_list),
+                    np.percentile(fit_list, 5), np.percentile(fit_list, 95)))
+            log_file.flush()
+    cm.__save_archive(nets_archive, gen_i,name="net")
+    cm.__save_archive(prompt_archive, gen_i,name="prompt")
+    return nets_archive,prompt_archive
+
+if __name__ == "__main__":
+    main()
+

LLMatic-main/logs/gen-nets_codegen-6B-mono_networks-10_temp-0.0_net-training-epochs-50_niches-100_infer-and-flops-as-bd/exp_results.csv

@@ -0,0 +1,4 @@
+generations,best_loss,used_prompt
+generations,best_loss,used_prompt
+generations,best_loss,used_prompt
+generations,best_loss,used_prompt

LLMatic-main/map_elites/LICENSE

@@ -0,0 +1,518 @@
+
+  CeCILL FREE SOFTWARE LICENSE AGREEMENT
+
+Version 2.1 dated 2013-06-21
+
+
+    Notice
+
+This Agreement is a Free Software license agreement that is the result
+of discussions between its authors in order to ensure compliance with
+the two main principles guiding its drafting:
+
+  * firstly, compliance with the principles governing the distribution
+    of Free Software: access to source code, broad rights granted to users,
+  * secondly, the election of a governing law, French law, with which it
+    is conformant, both as regards the law of torts and intellectual
+    property law, and the protection that it offers to both authors and
+    holders of the economic rights over software.
+
+The authors of the CeCILL (for Ce[a] C[nrs] I[nria] L[ogiciel] L[ibre]) 
+license are: 
+
+Commissariat � l'�nergie atomique et aux �nergies alternatives - CEA, a
+public scientific, technical and industrial research establishment,
+having its principal place of business at 25 rue Leblanc, immeuble Le
+Ponant D, 75015 Paris, France.
+
+Centre National de la Recherche Scientifique - CNRS, a public scientific
+and technological establishment, having its principal place of business
+at 3 rue Michel-Ange, 75794 Paris cedex 16, France.
+
+Institut National de Recherche en Informatique et en Automatique -
+Inria, a public scientific and technological establishment, having its
+principal place of business at Domaine de Voluceau, Rocquencourt, BP
+105, 78153 Le Chesnay cedex, France.
+
+
+    Preamble
+
+The purpose of this Free Software license agreement is to grant users
+the right to modify and redistribute the software governed by this
+license within the framework of an open source distribution model.
+
+The exercising of this right is conditional upon certain obligations for
+users so as to preserve this status for all subsequent redistributions.
+
+In consideration of access to the source code and the rights to copy,
+modify and redistribute granted by the license, users are provided only
+with a limited warranty and the software's author, the holder of the
+economic rights, and the successive licensors only have limited liability.
+
+In this respect, the risks associated with loading, using, modifying
+and/or developing or reproducing the software by the user are brought to
+the user's attention, given its Free Software status, which may make it
+complicated to use, with the result that its use is reserved for
+developers and experienced professionals having in-depth computer
+knowledge. Users are therefore encouraged to load and test the
+suitability of the software as regards their requirements in conditions
+enabling the security of their systems and/or data to be ensured and,
+more generally, to use and operate it in the same conditions of
+security. This Agreement may be freely reproduced and published,
+provided it is not altered, and that no provisions are either added or
+removed herefrom.
+
+This Agreement may apply to any or all software for which the holder of
+the economic rights decides to submit the use thereof to its provisions.
+
+Frequently asked questions can be found on the official website of the
+CeCILL licenses family (http://www.cecill.info/index.en.html) for any 
+necessary clarification.
+
+
+    Article 1 - DEFINITIONS
+
+For the purpose of this Agreement, when the following expressions
+commence with a capital letter, they shall have the following meaning:
+
+Agreement: means this license agreement, and its possible subsequent
+versions and annexes.
+
+Software: means the software in its Object Code and/or Source Code form
+and, where applicable, its documentation, "as is" when the Licensee
+accepts the Agreement.
+
+Initial Software: means the Software in its Source Code and possibly its
+Object Code form and, where applicable, its documentation, "as is" when
+it is first distributed under the terms and conditions of the Agreement.
+
+Modified Software: means the Software modified by at least one
+Contribution.
+
+Source Code: means all the Software's instructions and program lines to
+which access is required so as to modify the Software.
+
+Object Code: means the binary files originating from the compilation of
+the Source Code.
+
+Holder: means the holder(s) of the economic rights over the Initial
+Software.
+
+Licensee: means the Software user(s) having accepted the Agreement.
+
+Contributor: means a Licensee having made at least one Contribution.
+
+Licensor: means the Holder, or any other individual or legal entity, who
+distributes the Software under the Agreement.
+
+Contribution: means any or all modifications, corrections, translations,
+adaptations and/or new functions integrated into the Software by any or
+all Contributors, as well as any or all Internal Modules.
+
+Module: means a set of sources files including their documentation that
+enables supplementary functions or services in addition to those offered
+by the Software.
+
+External Module: means any or all Modules, not derived from the
+Software, so that this Module and the Software run in separate address
+spaces, with one calling the other when they are run.
+
+Internal Module: means any or all Module, connected to the Software so
+that they both execute in the same address space.
+
+GNU GPL: means the GNU General Public License version 2 or any
+subsequent version, as published by the Free Software Foundation Inc.
+
+GNU Affero GPL: means the GNU Affero General Public License version 3 or
+any subsequent version, as published by the Free Software Foundation Inc.
+
+EUPL: means the European Union Public License version 1.1 or any
+subsequent version, as published by the European Commission.
+
+Parties: mean both the Licensee and the Licensor.
+
+These expressions may be used both in singular and plural form.
+
+
+    Article 2 - PURPOSE
+
+The purpose of the Agreement is the grant by the Licensor to the
+Licensee of a non-exclusive, transferable and worldwide license for the
+Software as set forth in Article 5 <#scope> hereinafter for the whole
+term of the protection granted by the rights over said Software.
+
+
+    Article 3 - ACCEPTANCE
+
+3.1 The Licensee shall be deemed as having accepted the terms and
+conditions of this Agreement upon the occurrence of the first of the
+following events:
+
+  * (i) loading the Software by any or all means, notably, by
+    downloading from a remote server, or by loading from a physical medium;
+  * (ii) the first time the Licensee exercises any of the rights granted
+    hereunder.
+
+3.2 One copy of the Agreement, containing a notice relating to the
+characteristics of the Software, to the limited warranty, and to the
+fact that its use is restricted to experienced users has been provided
+to the Licensee prior to its acceptance as set forth in Article 3.1
+<#accepting> hereinabove, and the Licensee hereby acknowledges that it
+has read and understood it.
+
+
+    Article 4 - EFFECTIVE DATE AND TERM
+
+
+      4.1 EFFECTIVE DATE
+
+The Agreement shall become effective on the date when it is accepted by
+the Licensee as set forth in Article 3.1 <#accepting>.
+
+
+      4.2 TERM
+
+The Agreement shall remain in force for the entire legal term of
+protection of the economic rights over the Software.
+
+
+    Article 5 - SCOPE OF RIGHTS GRANTED
+
+The Licensor hereby grants to the Licensee, who accepts, the following
+rights over the Software for any or all use, and for the term of the
+Agreement, on the basis of the terms and conditions set forth hereinafter.
+
+Besides, if the Licensor owns or comes to own one or more patents
+protecting all or part of the functions of the Software or of its
+components, the Licensor undertakes not to enforce the rights granted by
+these patents against successive Licensees using, exploiting or
+modifying the Software. If these patents are transferred, the Licensor
+undertakes to have the transferees subscribe to the obligations set
+forth in this paragraph.
+
+
+      5.1 RIGHT OF USE
+
+The Licensee is authorized to use the Software, without any limitation
+as to its fields of application, with it being hereinafter specified
+that this comprises:
+
+ 1. permanent or temporary reproduction of all or part of the Software
+    by any or all means and in any or all form.
+
+ 2. loading, displaying, running, or storing the Software on any or all
+    medium.
+
+ 3. entitlement to observe, study or test its operation so as to
+    determine the ideas and principles behind any or all constituent
+    elements of said Software. This shall apply when the Licensee
+    carries out any or all loading, displaying, running, transmission or
+    storage operation as regards the Software, that it is entitled to
+    carry out hereunder.
+
+
+      5.2 ENTITLEMENT TO MAKE CONTRIBUTIONS
+
+The right to make Contributions includes the right to translate, adapt,
+arrange, or make any or all modifications to the Software, and the right
+to reproduce the resulting software.
+
+The Licensee is authorized to make any or all Contributions to the
+Software provided that it includes an explicit notice that it is the
+author of said Contribution and indicates the date of the creation thereof.
+
+
+      5.3 RIGHT OF DISTRIBUTION
+
+In particular, the right of distribution includes the right to publish,
+transmit and communicate the Software to the general public on any or
+all medium, and by any or all means, and the right to market, either in
+consideration of a fee, or free of charge, one or more copies of the
+Software by any means.
+
+The Licensee is further authorized to distribute copies of the modified
+or unmodified Software to third parties according to the terms and
+conditions set forth hereinafter.
+
+
+        5.3.1 DISTRIBUTION OF SOFTWARE WITHOUT MODIFICATION
+
+The Licensee is authorized to distribute true copies of the Software in
+Source Code or Object Code form, provided that said distribution
+complies with all the provisions of the Agreement and is accompanied by:
+
+ 1. a copy of the Agreement,
+
+ 2. a notice relating to the limitation of both the Licensor's warranty
+    and liability as set forth in Articles 8 and 9,
+
+and that, in the event that only the Object Code of the Software is
+redistributed, the Licensee allows effective access to the full Source
+Code of the Software for a period of at least three years from the
+distribution of the Software, it being understood that the additional
+acquisition cost of the Source Code shall not exceed the cost of the
+data transfer.
+
+
+        5.3.2 DISTRIBUTION OF MODIFIED SOFTWARE
+
+When the Licensee makes a Contribution to the Software, the terms and
+conditions for the distribution of the resulting Modified Software
+become subject to all the provisions of this Agreement.
+
+The Licensee is authorized to distribute the Modified Software, in
+source code or object code form, provided that said distribution
+complies with all the provisions of the Agreement and is accompanied by:
+
+ 1. a copy of the Agreement,
+
+ 2. a notice relating to the limitation of both the Licensor's warranty
+    and liability as set forth in Articles 8 and 9,
+
+and, in the event that only the object code of the Modified Software is
+redistributed,
+
+ 3. a note stating the conditions of effective access to the full source
+    code of the Modified Software for a period of at least three years
+    from the distribution of the Modified Software, it being understood
+    that the additional acquisition cost of the source code shall not
+    exceed the cost of the data transfer.
+
+
+        5.3.3 DISTRIBUTION OF EXTERNAL MODULES
+
+When the Licensee has developed an External Module, the terms and
+conditions of this Agreement do not apply to said External Module, that
+may be distributed under a separate license agreement.
+
+
+        5.3.4 COMPATIBILITY WITH OTHER LICENSES
+
+The Licensee can include a code that is subject to the provisions of one
+of the versions of the GNU GPL, GNU Affero GPL and/or EUPL in the
+Modified or unmodified Software, and distribute that entire code under
+the terms of the same version of the GNU GPL, GNU Affero GPL and/or EUPL.
+
+The Licensee can include the Modified or unmodified Software in a code
+that is subject to the provisions of one of the versions of the GNU GPL,
+GNU Affero GPL and/or EUPL and distribute that entire code under the
+terms of the same version of the GNU GPL, GNU Affero GPL and/or EUPL.
+
+
+    Article 6 - INTELLECTUAL PROPERTY
+
+
+      6.1 OVER THE INITIAL SOFTWARE
+
+The Holder owns the economic rights over the Initial Software. Any or
+all use of the Initial Software is subject to compliance with the terms
+and conditions under which the Holder has elected to distribute its work
+and no one shall be entitled to modify the terms and conditions for the
+distribution of said Initial Software.
+
+The Holder undertakes that the Initial Software will remain ruled at
+least by this Agreement, for the duration set forth in Article 4.2 <#term>.
+
+
+      6.2 OVER THE CONTRIBUTIONS
+
+The Licensee who develops a Contribution is the owner of the
+intellectual property rights over this Contribution as defined by
+applicable law.
+
+
+      6.3 OVER THE EXTERNAL MODULES
+
+The Licensee who develops an External Module is the owner of the
+intellectual property rights over this External Module as defined by
+applicable law and is free to choose the type of agreement that shall
+govern its distribution.
+
+
+      6.4 JOINT PROVISIONS
+
+The Licensee expressly undertakes:
+
+ 1. not to remove, or modify, in any manner, the intellectual property
+    notices attached to the Software;
+
+ 2. to reproduce said notices, in an identical manner, in the copies of
+    the Software modified or not.
+
+The Licensee undertakes not to directly or indirectly infringe the
+intellectual property rights on the Software of the Holder and/or
+Contributors, and to take, where applicable, vis-�-vis its staff, any
+and all measures required to ensure respect of said intellectual
+property rights of the Holder and/or Contributors.
+
+
+    Article 7 - RELATED SERVICES
+
+7.1 Under no circumstances shall the Agreement oblige the Licensor to
+provide technical assistance or maintenance services for the Software.
+
+However, the Licensor is entitled to offer this type of services. The
+terms and conditions of such technical assistance, and/or such
+maintenance, shall be set forth in a separate instrument. Only the
+Licensor offering said maintenance and/or technical assistance services
+shall incur liability therefor.
+
+7.2 Similarly, any Licensor is entitled to offer to its licensees, under
+its sole responsibility, a warranty, that shall only be binding upon
+itself, for the redistribution of the Software and/or the Modified
+Software, under terms and conditions that it is free to decide. Said
+warranty, and the financial terms and conditions of its application,
+shall be subject of a separate instrument executed between the Licensor
+and the Licensee.
+
+
+    Article 8 - LIABILITY
+
+8.1 Subject to the provisions of Article 8.2, the Licensee shall be
+entitled to claim compensation for any direct loss it may have suffered
+from the Software as a result of a fault on the part of the relevant
+Licensor, subject to providing evidence thereof.
+
+8.2 The Licensor's liability is limited to the commitments made under
+this Agreement and shall not be incurred as a result of in particular:
+(i) loss due the Licensee's total or partial failure to fulfill its
+obligations, (ii) direct or consequential loss that is suffered by the
+Licensee due to the use or performance of the Software, and (iii) more
+generally, any consequential loss. In particular the Parties expressly
+agree that any or all pecuniary or business loss (i.e. loss of data,
+loss of profits, operating loss, loss of customers or orders,
+opportunity cost, any disturbance to business activities) or any or all
+legal proceedings instituted against the Licensee by a third party,
+shall constitute consequential loss and shall not provide entitlement to
+any or all compensation from the Licensor.
+
+
+    Article 9 - WARRANTY
+
+9.1 The Licensee acknowledges that the scientific and technical
+state-of-the-art when the Software was distributed did not enable all
+possible uses to be tested and verified, nor for the presence of
+possible defects to be detected. In this respect, the Licensee's
+attention has been drawn to the risks associated with loading, using,
+modifying and/or developing and reproducing the Software which are
+reserved for experienced users.
+
+The Licensee shall be responsible for verifying, by any or all means,
+the suitability of the product for its requirements, its good working
+order, and for ensuring that it shall not cause damage to either persons
+or properties.
+
+9.2 The Licensor hereby represents, in good faith, that it is entitled
+to grant all the rights over the Software (including in particular the
+rights set forth in Article 5 <#scope>).
+
+9.3 The Licensee acknowledges that the Software is supplied "as is" by
+the Licensor without any other express or tacit warranty, other than
+that provided for in Article 9.2 <#good-faith> and, in particular,
+without any warranty as to its commercial value, its secured, safe,
+innovative or relevant nature.
+
+Specifically, the Licensor does not warrant that the Software is free
+from any error, that it will operate without interruption, that it will
+be compatible with the Licensee's own equipment and software
+configuration, nor that it will meet the Licensee's requirements.
+
+9.4 The Licensor does not either expressly or tacitly warrant that the
+Software does not infringe any third party intellectual property right
+relating to a patent, software or any other property right. Therefore,
+the Licensor disclaims any and all liability towards the Licensee
+arising out of any or all proceedings for infringement that may be
+instituted in respect of the use, modification and redistribution of the
+Software. Nevertheless, should such proceedings be instituted against
+the Licensee, the Licensor shall provide it with technical and legal
+expertise for its defense. Such technical and legal expertise shall be
+decided on a case-by-case basis between the relevant Licensor and the
+Licensee pursuant to a memorandum of understanding. The Licensor
+disclaims any and all liability as regards the Licensee's use of the
+name of the Software. No warranty is given as regards the existence of
+prior rights over the name of the Software or as regards the existence
+of a trademark.
+
+
+    Article 10 - TERMINATION
+
+10.1 In the event of a breach by the Licensee of its obligations
+hereunder, the Licensor may automatically terminate this Agreement
+thirty (30) days after notice has been sent to the Licensee and has
+remained ineffective.
+
+10.2 A Licensee whose Agreement is terminated shall no longer be
+authorized to use, modify or distribute the Software. However, any
+licenses that it may have granted prior to termination of the Agreement
+shall remain valid subject to their having been granted in compliance
+with the terms and conditions hereof.
+
+
+    Article 11 - MISCELLANEOUS
+
+
+      11.1 EXCUSABLE EVENTS
+
+Neither Party shall be liable for any or all delay, or failure to
+perform the Agreement, that may be attributable to an event of force
+majeure, an act of God or an outside cause, such as defective
+functioning or interruptions of the electricity or telecommunications
+networks, network paralysis following a virus attack, intervention by
+government authorities, natural disasters, water damage, earthquakes,
+fire, explosions, strikes and labor unrest, war, etc.
+
+11.2 Any failure by either Party, on one or more occasions, to invoke
+one or more of the provisions hereof, shall under no circumstances be
+interpreted as being a waiver by the interested Party of its right to
+invoke said provision(s) subsequently.
+
+11.3 The Agreement cancels and replaces any or all previous agreements,
+whether written or oral, between the Parties and having the same
+purpose, and constitutes the entirety of the agreement between said
+Parties concerning said purpose. No supplement or modification to the
+terms and conditions hereof shall be effective as between the Parties
+unless it is made in writing and signed by their duly authorized
+representatives.
+
+11.4 In the event that one or more of the provisions hereof were to
+conflict with a current or future applicable act or legislative text,
+said act or legislative text shall prevail, and the Parties shall make
+the necessary amendments so as to comply with said act or legislative
+text. All other provisions shall remain effective. Similarly, invalidity
+of a provision of the Agreement, for any reason whatsoever, shall not
+cause the Agreement as a whole to be invalid.
+
+
+      11.5 LANGUAGE
+
+The Agreement is drafted in both French and English and both versions
+are deemed authentic.
+
+
+    Article 12 - NEW VERSIONS OF THE AGREEMENT
+
+12.1 Any person is authorized to duplicate and distribute copies of this
+Agreement.
+
+12.2 So as to ensure coherence, the wording of this Agreement is
+protected and may only be modified by the authors of the License, who
+reserve the right to periodically publish updates or new versions of the
+Agreement, each with a separate number. These subsequent versions may
+address new issues encountered by Free Software.
+
+12.3 Any Software distributed under a given version of the Agreement may
+only be subsequently distributed under the same version of the Agreement
+or a subsequent version, subject to the provisions of Article 5.3.4
+<#compatibility>.
+
+
+    Article 13 - GOVERNING LAW AND JURISDICTION
+
+13.1 The Agreement is governed by French law. The Parties agree to
+endeavor to seek an amicable solution to any disagreements or disputes
+that may arise during the performance of the Agreement.
+
+13.2 Failing an amicable solution within two (2) months as from their
+occurrence, and unless emergency proceedings are necessary, the
+disagreements or disputes shall be referred to the Paris Courts having
+jurisdiction, by the more diligent Party.

LLMatic-main/map_elites/common.py

@@ -0,0 +1,231 @@
+#! /usr/bin/env python
+#| This file is a part of the pymap_elites framework.
+#| Copyright 2019, INRIA
+#| Main contributor(s):
+#| Jean-Baptiste Mouret, jean-baptiste.mouret@inria.fr
+#| Eloise Dalin , eloise.dalin@inria.fr
+#| Pierre Desreumaux , pierre.desreumaux@inria.fr
+#|
+#|
+#| **Main paper**: Mouret JB, Clune J. Illuminating search spaces by
+#| mapping elites. arXiv preprint arXiv:1504.04909. 2015 Apr 20.
+#|
+#| This software is governed by the CeCILL license under French law
+#| and abiding by the rules of distribution of free software.  You
+#| can use, modify and/ or redistribute the software under the terms
+#| of the CeCILL license as circulated by CEA, CNRS and INRIA at the
+#| following URL "http://www.cecill.info".
+#|
+#| As a counterpart to the access to the source code and rights to
+#| copy, modify and redistribute granted by the license, users are
+#| provided only with a limited warranty and the software's author,
+#| the holder of the economic rights, and the successive licensors
+#| have only limited liability.
+#|
+#| In this respect, the user's attention is drawn to the risks
+#| associated with loading, using, modifying and/or developing or
+#| reproducing the software by the user in light of its specific
+#| status of free software, that may mean that it is complicated to
+#| manipulate, and that also therefore means that it is reserved for
+#| developers and experienced professionals having in-depth computer
+#| knowledge. Users are therefore encouraged to load and test the
+#| software's suitability as regards their requirements in conditions
+#| enabling the security of their systems and/or data to be ensured
+#| and, more generally, to use and operate it in the same conditions
+#| as regards security.
+#|
+#| The fact that you are presently reading this means that you have
+#| had knowledge of the CeCILL license and that you accept its terms.
+#
+
+import math
+import numpy as np
+import multiprocessing
+from pathlib import Path
+import sys
+import random
+from collections import defaultdict
+from sklearn.cluster import KMeans
+
+default_params = \
+    {
+        # more of this -> higher-quality CVT
+        "cvt_samples": 25000,
+        # we evaluate in batches to paralleliez
+        "batch_size": 100,
+        # proportion of niches to be filled before starting
+        "random_init": 0.1,
+        # batch for random initialization
+        "random_init_batch": 100,
+        # when to write results (one generation = one batch)
+        "dump_period": 10000,
+        # do we use several cores?
+        "parallel": True,
+        # do we cache the result of CVT and reuse?
+        "cvt_use_cache": True,
+        # min/max of parameters
+        "min": 0,
+        "max": 1,
+        # only useful if you use the 'iso_dd' variation operator
+        "iso_sigma": 0.01,
+        "line_sigma": 0.2
+    }
+
+class Species:
+    def __init__(self, x, desc, fitness, net_path, centroid=None):
+        self.x = x
+        self.desc = desc
+        self.fitness = fitness
+        self.centroid = centroid
+        self.curiosity = 0
+        self.net_path = net_path
+
+
+def polynomial_mutation(x):
+    '''
+    Cf Deb 2001, p 124 ; param: eta_m
+    '''
+    y = x.copy()
+    eta_m = 5.0;
+    r = np.random.random(size=len(x))
+    for i in range(0, len(x)):
+        if r[i] < 0.5:
+            delta_i = math.pow(2.0 * r[i], 1.0 / (eta_m + 1.0)) - 1.0
+        else:
+            delta_i = 1 - math.pow(2.0 * (1.0 - r[i]), 1.0 / (eta_m + 1.0))
+        y[i] += delta_i
+    return y
+
+def sbx(x, y, params):
+    '''
+    SBX (cf Deb 2001, p 113) Simulated Binary Crossover
+
+    A large value ef eta gives a higher probablitity for
+    creating a `near-parent' solutions and a small value allows
+    distant solutions to be selected as offspring.
+    '''
+    eta = 10.0
+    xl = params['min']
+    xu = params['max']
+    z = x.copy()
+    r1 = np.random.random(size=len(x))
+    r2 = np.random.random(size=len(x))
+
+    for i in range(0, len(x)):
+        if abs(x[i] - y[i]) > 1e-15:
+            x1 = min(x[i], y[i])
+            x2 = max(x[i], y[i])
+
+            beta = 1.0 + (2.0 * (x1 - xl) / (x2 - x1))
+            alpha = 2.0 - beta ** -(eta + 1)
+            rand = r1[i]
+            if rand <= 1.0 / alpha:
+                beta_q = (rand * alpha) ** (1.0 / (eta + 1))
+            else:
+                beta_q = (1.0 / (2.0 - rand * alpha)) ** (1.0 / (eta + 1))
+
+            c1 = 0.5 * (x1 + x2 - beta_q * (x2 - x1))
+
+            beta = 1.0 + (2.0 * (xu - x2) / (x2 - x1))
+            alpha = 2.0 - beta ** -(eta + 1)
+            if rand <= 1.0 / alpha:
+                beta_q = (rand * alpha) ** (1.0 / (eta + 1))
+            else:
+                beta_q = (1.0 / (2.0 - rand * alpha)) ** (1.0 / (eta + 1))
+            c2 = 0.5 * (x1 + x2 + beta_q * (x2 - x1))
+
+            c1 = min(max(c1, xl), xu)
+            c2 = min(max(c2, xl), xu)
+
+            if r2[i] <= 0.5:
+                z[i] = c2
+            else:
+                z[i] = c1
+    return z
+
+
+def iso_dd(x, y, params):
+    '''
+    Iso+Line
+    Ref:
+    Vassiliades V, Mouret JB. Discovering the elite hypervolume by leveraging interspecies correlation.
+    GECCO 2018
+    '''
+    assert(x.shape == y.shape)
+    p_max = np.array(params["max"])
+    p_min = np.array(params["min"])
+    a = np.random.normal(0, params['iso_sigma'], size=len(x))
+    b = np.random.normal(0, params['line_sigma'])
+    norm = np.linalg.norm(x - y)
+    z = x.copy() + a + b * (x - y)
+    return np.clip(z, p_min, p_max)
+
+
+def variation(x, z, params):
+    assert(x.shape == z.shape)
+    y = sbx(x, z, params)
+    return y
+
+def __centroids_filename(k, dim):
+    return 'centroids_' + str(k) + '_' + str(dim) + '.dat'
+
+
+def __write_centroids(centroids):
+    k = centroids.shape[0]
+    dim = centroids.shape[1]
+    filename = __centroids_filename(k, dim)
+    with open(filename, 'w') as f:
+        for p in centroids:
+            for item in p:
+                f.write(str(item) + ' ')
+            f.write('\n')
+
+
+def cvt(k, dim, samples, cvt_use_cache=True):
+    # check if we have cached values
+    fname = __centroids_filename(k, dim)
+    if cvt_use_cache:
+        if Path(fname).is_file():
+            print("WARNING: using cached CVT:", fname)
+            return np.loadtxt(fname)
+    # otherwise, compute cvt
+    print("Computing CVT (this can take a while...):", fname)
+
+    x = np.random.rand(samples, dim)
+    k_means = KMeans(init='k-means++', n_clusters=k,
+                     n_init=1, verbose=1)#,algorithm="full")
+    k_means.fit(x)
+    __write_centroids(k_means.cluster_centers_)
+
+    return k_means.cluster_centers_
+
+
+def make_hashable(array):
+    return tuple(map(float, array))
+
+
+def parallel_eval(evaluate_function, to_evaluate, pool, params):
+    if params['parallel'] == True:
+        s_list = pool.map(evaluate_function, to_evaluate)
+    else:
+        s_list = map(evaluate_function, to_evaluate)
+    return list(s_list)
+
+# format: fitness, centroid, desc, genome \n
+# fitness, centroid, desc and x are vectors
+def __save_archive(archive, gen,name="net"):
+    def write_array(a, f):
+        for i in a:
+            f.write(str(i) + " ")
+    filename = name+'archive_' + str(gen) + '.dat'
+    with open(filename, 'w') as f:
+        for k in archive.values():
+            f.write(k.net_path + " ")
+            f.write(str(k.fitness) + ' ')
+            write_array(k.centroid, f)
+            f.write(str(k.curiosity) + " ")
+            f.write(str(k.desc[0]) + " ")
+            f.write(str(k.desc[1]) + " ")
+            
+            #write_array(k.x, f)
+            f.write("\n")

LLMatic-main/map_elites/cvt.py

@@ -0,0 +1,142 @@
+#! /usr/bin/env python
+#| This file is a part of the pymap_elites framework.
+#| Copyright 2019, INRIA
+#| Main contributor(s):
+#| Jean-Baptiste Mouret, jean-baptiste.mouret@inria.fr
+#| Eloise Dalin , eloise.dalin@inria.fr
+#| Pierre Desreumaux , pierre.desreumaux@inria.fr
+#|
+#|
+#| **Main paper**: Mouret JB, Clune J. Illuminating search spaces by
+#| mapping elites. arXiv preprint arXiv:1504.04909. 2015 Apr 20.
+#|
+#| This software is governed by the CeCILL license under French law
+#| and abiding by the rules of distribution of free software.  You
+#| can use, modify and/ or redistribute the software under the terms
+#| of the CeCILL license as circulated by CEA, CNRS and INRIA at the
+#| following URL "http://www.cecill.info".
+#|
+#| As a counterpart to the access to the source code and rights to
+#| copy, modify and redistribute granted by the license, users are
+#| provided only with a limited warranty and the software's author,
+#| the holder of the economic rights, and the successive licensors
+#| have only limited liability.
+#|
+#| In this respect, the user's attention is drawn to the risks
+#| associated with loading, using, modifying and/or developing or
+#| reproducing the software by the user in light of its specific
+#| status of free software, that may mean that it is complicated to
+#| manipulate, and that also therefore means that it is reserved for
+#| developers and experienced professionals having in-depth computer
+#| knowledge. Users are therefore encouraged to load and test the
+#| software's suitability as regards their requirements in conditions
+#| enabling the security of their systems and/or data to be ensured
+#| and, more generally, to use and operate it in the same conditions
+#| as regards security.
+#|
+#| The fact that you are presently reading this means that you have
+#| had knowledge of the CeCILL license and that you accept its terms.
+
+import math
+import numpy as np
+import multiprocessing
+
+# from scipy.spatial import cKDTree : TODO -- faster?
+from sklearn.neighbors import KDTree
+
+from map_elites import common as cm
+
+
+
+def __add_to_archive(s, centroid, archive, kdt):
+    niche_index = kdt.query([centroid], k=1)[1][0][0]
+    niche = kdt.data[niche_index]
+    n = cm.make_hashable(niche)
+    s.centroid = n
+    if n in archive:
+        if s.fitness > archive[n].fitness:
+            archive[n] = s
+            return 1
+        return 0
+    else:
+        archive[n] = s
+        return 1
+
+
+# evaluate a single vector (x) with a function f and return a species
+# t = vector, function
+def __evaluate(t):
+    z, f = t  # evaluate z with function f
+    fit, desc = f(z)
+    return cm.Species(z, desc, fit)
+
+# map-elites algorithm (CVT variant)
+def compute(dim_map, dim_x, f,
+            n_niches=1000,
+            max_evals=1e5,
+            params=cm.default_params,
+            log_file=None,
+            variation_operator=cm.variation):
+    """CVT MAP-Elites
+       Vassiliades V, Chatzilygeroudis K, Mouret JB. Using centroidal voronoi tessellations to scale up the multidimensional archive of phenotypic elites algorithm. IEEE Transactions on Evolutionary Computation. 2017 Aug 3;22(4):623-30.
+
+       Format of the logfile: evals archive_size max mean median 5%_percentile, 95%_percentile
+
+    """
+    # setup the parallel processing pool
+    num_cores = multiprocessing.cpu_count()
+    pool = multiprocessing.Pool(num_cores)
+
+    # create the CVT
+    c = cm.cvt(n_niches, dim_map,
+              params['cvt_samples'], params['cvt_use_cache'])
+    kdt = KDTree(c, leaf_size=30, metric='euclidean')
+    cm.__write_centroids(c)
+
+    archive = {} # init archive (empty)
+    n_evals = 0 # number of evaluations since the beginning
+    b_evals = 0 # number evaluation since the last dump
+
+    # main loop
+    while (n_evals < max_evals):
+        to_evaluate = []
+        # random initialization
+        if len(archive) <= params['random_init'] * n_niches:
+            for i in range(0, params['random_init_batch']):
+                x = np.random.uniform(low=params['min'], high=params['max'], size=dim_x)
+                to_evaluate += [(x, f)]
+        else:  # variation/selection loop
+            keys = list(archive.keys())
+            # we select all the parents at the same time because randint is slow
+            rand1 = np.random.randint(len(keys), size=params['batch_size'])
+            rand2 = np.random.randint(len(keys), size=params['batch_size'])
+            for n in range(0, params['batch_size']):
+                # parent selection
+                x = archive[keys[rand1[n]]]
+                y = archive[keys[rand2[n]]]
+                # copy & add variation
+                z = variation_operator(x.x, y.x, params)
+                to_evaluate += [(z, f)]
+        # evaluation of the fitness for to_evaluate
+        s_list = cm.parallel_eval(__evaluate, to_evaluate, pool, params)
+        # natural selection
+        for s in s_list:
+            __add_to_archive(s, s.desc, archive, kdt)
+        # count evals
+        n_evals += len(to_evaluate)
+        b_evals += len(to_evaluate)
+
+        # write archive
+        if b_evals >= params['dump_period'] and params['dump_period'] != -1:
+            print("[{}/{}]".format(n_evals, int(max_evals)), end=" ", flush=True)
+            cm.__save_archive(archive, n_evals)
+            b_evals = 0
+        # write log
+        if log_file != None:
+            fit_list = np.array([x.fitness for x in archive.values()])
+            log_file.write("{} {} {} {} {} {} {}\n".format(n_evals, len(archive.keys()),
+                    fit_list.max(), np.mean(fit_list), np.median(fit_list),
+                    np.percentile(fit_list, 5), np.percentile(fit_list, 95)))
+            log_file.flush()
+    cm.__save_archive(archive, n_evals)
+    return archive

LLMatic-main/map_elites/multitask.py

@@ -0,0 +1,242 @@
+#! /usr/bin/env python
+#| This file is a part of the pymap_elites framework.
+#| Copyright 2019, INRIA
+#| Main contributor(s):
+#| Jean-Baptiste Mouret, jean-baptiste.mouret@inria.fr
+#| Eloise Dalin , eloise.dalin@inria.fr
+#| Pierre Desreumaux , pierre.desreumaux@inria.fr
+#|
+#|
+#| **Main paper**: Mouret JB, Clune J. Illuminating search spaces by
+#| mapping elites. arXiv preprint arXiv:1504.04909. 2015 Apr 20.
+#|
+#| This software is governed by the CeCILL license under French law
+#| and abiding by the rules of distribution of free software.  You
+#| can use, modify and/ or redistribute the software under the terms
+#| of the CeCILL license as circulated by CEA, CNRS and INRIA at the
+#| following URL "http://www.cecill.info".
+#|
+#| As a counterpart to the access to the source code and rights to
+#| copy, modify and redistribute granted by the license, users are
+#| provided only with a limited warranty and the software's author,
+#| the holder of the economic rights, and the successive licensors
+#| have only limited liability.
+#|
+#| In this respect, the user's attention is drawn to the risks
+#| associated with loading, using, modifying and/or developing or
+#| reproducing the software by the user in light of its specific
+#| status of free software, that may mean that it is complicated to
+#| manipulate, and that also therefore means that it is reserved for
+#| developers and experienced professionals having in-depth computer
+#| knowledge. Users are therefore encouraged to load and test the
+#| software's suitability as regards their requirements in conditions
+#| enabling the security of their systems and/or data to be ensured
+#| and, more generally, to use and operate it in the same conditions
+#| as regards security.
+#|
+#| The fact that you are presently reading this means that you have
+#| had knowledge of the CeCILL license and that you accept its terms.
+#
+# from scipy.spatial import cKDTree : TODO
+
+import math
+import numpy as np
+import multiprocessing
+from pathlib import Path
+import sys
+import random
+from collections import defaultdict
+from sklearn.neighbors import KDTree
+from scipy.spatial import distance
+
+from map_elites import common as cm
+
+
+def add_to_archive(s, archive):
+    centroid = cm.make_hashable(s.centroid)
+    if centroid in archive:
+        if s.fitness > archive[centroid].fitness:
+            archive[centroid] = s
+            return 1
+        return 0
+    else:
+        archive[centroid] = s
+        return 1
+
+# evaluate a single vector (z) with a function f and return a species
+# t = vector, function
+def __evaluate(t):
+    z, f, task, centroid, _ = t
+    fit = f(z, task)
+    return cm.Species(z, task, fit, centroid)
+
+# bandit opt for optimizing tournament size
+# probability matching / Adaptive pursuit Thierens GECCO 2005
+# UCB: schoenauer / Sebag
+# TODO : params for values, and params for window
+def bandit(successes, n_niches):
+    n = 0
+    for v in successes.values():
+        n += len(v)
+    v = [1, 10, 50, 100, 500]#, 1000]
+    if len(successes.keys()) < len(v):
+        return random.choice(v)
+    ucb = []
+    for k in v:
+        x = [i[0] for i in successes[k]]
+        mean = sum(x) / float(len(x)) # 100 = batch size??
+        n_a = len(x)
+        ucb += [mean +  math.sqrt(2 * math.log(n) / n_a)]
+    a = np.argmax(ucb)
+    t_size = v[a]
+    return t_size
+
+# select the niche according to
+def select_niche(x, z, f, centroids, tasks, t_size, params, use_distance=False):
+    to_evaluate = []
+    if not use_distance:
+        # No distance: evaluate on a random niche
+        niche = np.random.randint(len(tasks))
+        to_evaluate += [(z, f, tasks[niche], centroids[niche, :], params)]
+    else:
+        # we select the parent (a single one), then we select the niche
+        # with a tournament based on the task distance
+        # the size of the tournament depends on the bandit algorithm
+        niches_centroids = []
+        niches_tasks = [] # TODO : use a kd-tree
+        rand = np.random.randint(centroids.shape[0], size=t_size)
+        for p in range(0, t_size):
+            n = rand[p]
+            niches_centroids += [centroids[n, :]]
+            niches_tasks += [tasks[n]]
+        cd = distance.cdist(niches_centroids, [x.centroid], 'euclidean')
+        cd_min = np.argmin(cd)
+        to_evaluate += [(z, f, niches_tasks[cd_min], niches_centroids[cd_min], params)]
+    return to_evaluate
+
+
+def compute(dim_map=-1,
+            dim_x=-1,
+            f=None,
+            max_evals=1e5,
+            centroids=[],
+            tasks=[],
+            variation_operator=cm.variation,
+            params=cm.default_params,
+            log_file=None):
+    """Multi-task MAP-Elites
+    - if there is no centroid : random assignation of niches
+    - if there is no task: use the centroids as tasks
+    - if there is a centroid list: use the centroids to compute distances
+    when using the distance, use the bandit to select the tournament size (cf paper):
+
+    Format of the logfile: evals archive_size max mean 5%_percentile, 95%_percentile
+
+    Reference:
+    Mouret and Maguire (2020). Quality Diversity for Multitask Optimization
+    Proceedings of ACM GECCO.
+    """
+    print(params)
+    assert(f != None)
+    assert(dim_x != -1)
+    # handle the arguments
+    use_distance = False
+    if tasks != [] and centroids != []:
+        use_distance = True
+    elif tasks == [] and centroids != []:
+        # if no task, we use the centroids as tasks
+        tasks = centroids
+        use_distance = True
+    elif tasks != [] and centroids == []:
+        # if no centroid, we create indices so that we can index the archive by centroid
+        centroids = np.arange(0, len(tasks)).reshape(len(tasks), 1)
+        use_distance = False
+    else:
+        raise ValueError('Multi-task MAP-Elites: you need to specify a list of task, a list of centroids, or both')
+    print("Multitask-MAP-Elites:: using distance =>", use_distance)
+
+    assert(len(tasks) == len(centroids))
+    n_tasks = len(tasks)
+
+    # init archive (empty)
+    archive = {}
+
+    init_count = 0
+
+    # init multiprocessing
+    num_cores = multiprocessing.cpu_count()
+    pool = multiprocessing.Pool(num_cores)
+
+    # main loop
+    n_evals = 0 # number of evaluations
+    b_evals = 0 # number evaluation since the last dump
+    t_size = 1  # size of the tournament (if using distance) [will be selected by the bandit]
+    successes = defaultdict(list) # count the successes
+    while (n_evals < max_evals):
+        to_evaluate = []
+        to_evaluate_centroid = []
+        if len(archive) <= params['random_init'] * n_tasks:
+            # initialize the map with random individuals
+            for i in range(0, params['random_init_batch']):
+                # create a random individual
+                x = np.random.uniform(low=params['min'], high=params['max'], size=dim_x)
+                # we take a random task
+                n = np.random.randint(0, n_tasks)
+                to_evaluate += [(x, f, tasks[n], centroids[n], params)]
+            s_list = cm.parallel_eval(__evaluate, to_evaluate, pool, params)
+            n_evals += len(to_evaluate)
+            b_evals += len(to_evaluate)
+            for i in range(0, len(list(s_list))):
+                add_to_archive(s_list[i], archive)
+        else:
+            # main variation/selection loop
+            keys = list(archive.keys())
+            # we do all the randint together because randint is slow
+            rand1 = np.random.randint(len(keys), size=params['batch_size'])
+            rand2 = np.random.randint(len(keys), size=params['batch_size'])
+            for n in range(0, params['batch_size']):
+                # parent selection
+                x = archive[keys[rand1[n]]]
+                y = archive[keys[rand2[n]]]
+                # copy & add variation
+                z = variation_operator(x.x, y.x, params)
+                # different modes for multi-task (to select the niche)
+                to_evaluate += select_niche(x, z, f, centroids, tasks, t_size, params, use_distance)
+            # parallel evaluation of the fitness
+            s_list = cm.parallel_eval(__evaluate, to_evaluate, pool, params)
+            n_evals += len(to_evaluate)
+            b_evals += len(to_evaluate)
+            # natural selection
+            suc = 0
+            for i in range(0, len(list(s_list))):
+                suc += add_to_archive(s_list[i], archive)
+            if use_distance:
+                successes[t_size] += [(suc, n_evals)]
+        if use_distance: # call the bandit to optimize t_size
+            t_size = bandit(successes, n_tasks)
+
+        # write archive
+        if params['dump_period'] != -1 and b_evals > params['dump_period']:
+            cm.__save_archive(archive, n_evals)
+            b_evals = 0
+            n_e = [len(v) for v in successes.values()]
+            print(n_evals, n_e)
+            np.savetxt('t_size.dat', np.array(n_e))
+        if log_file != None:
+            fit_list = np.array([x.fitness for x in archive.values()])
+            log_file.write("{} {} {} {} {} {} {}\n".format(n_evals, len(archive.keys()), fit_list.max(), np.mean(fit_list), np.median(fit_list), np.percentile(fit_list, 5), np.percentile(fit_list, 95)))
+            log_file.flush()
+    cm.__save_archive(archive, n_evals)
+    return archive
+
+
+# a small test
+if __name__ == "__main__":
+    def rastrigin(xx):
+        x = xx * 10.0 - 5.0
+        f = 10 * x.shape[0]
+        for i in range(0, x.shape[0]):
+            f += x[i] * x[i] - 10 * math.cos(2 * math.pi * x[i])
+        return -f, np.array([xx[0], xx[1]])
+    # CVT-based version
+    my_map = compute(dim_map=2, dim_x = 10, n_niches=1500, f=rastrigin)

LLMatic-main/mutation_models.py

@@ -0,0 +1,73 @@
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, CodeGenForCausalLM
+#import openai
+import os
+import ray
+
+from conf.config import Config
+
+cfg = Config()
+
+
+def codex_mutate(cfg: Config, prompt, model="code-davinci-002",temperature=0.5):
+    
+    os.environ['OPENAI_API_KEY'] = ""# ENETR OPENAI API KEY HERE
+    openai.api_key = os.getenv("OPENAI_API_KEY")
+    return openai.Completion.create(
+                                    model=model,
+                                    prompt=prompt,
+                                    max_tokens=300,
+                                    temperature=temperature,
+                                    ) 
+
+
+#['codegen-350M-multi', 'codegen-2B-multi', 'codegen-6B-multi', 'codegen-16B-multi', 'codegen-350M-mono', 'codegen-2B-mono', 'codegen-6B-mono', 'codegen-16B-mono']
+#@ray.remote(num_cpus=cfg.NUM_CPUS)  # FIXME: Number of parallel processes should be configured globally.
+@ray.remote(num_gpus=cfg.NUM_GPUS)
+def codegen_mutate(cfg: Config, prompt, temperature):
+    diff = False
+
+    if diff:
+        # model = "/mnt/lustre/users/mnasir/NAS-LLM/diff-codegen-6B"
+        model = os.path.join(cfg.SAVE_DIR, "diff-codegen-6B")
+        tokenizer = AutoTokenizer.from_pretrained(model)#.to(cfg.DEVICE)
+        tokenizer.padding_side = 'left'
+        tokenizer.pad_token = tokenizer.eos_token
+        model = CodeGenForCausalLM.from_pretrained(model).to(cfg.DEVICE)
+        inputs = tokenizer(prompt, return_tensors='pt', padding=True)#.to(device)
+        #model.config.use_cache = True
+        sample = model.generate(**inputs, temperature=0.5, max_length=len(inputs[0]) + 300)
+        
+        return tokenizer.decode(sample[0][len(inputs[0]):])
+    else:
+        model = f'Salesforce/{cfg.MUTATION}'
+        # model = "/mnt/lustre/users/mnasir/NAS-LLM/codegen-6B"
+        #model = "/mnt/lustre/users/mnasir/NAS-LLM/diff-codegen-6B"
+        #model = os.path.join(cfg.SAVE_DIR, "codegen-6B")
+
+        # TODO: Should be doing something like this to download the model automatically.
+        # model = os.path.join(cfg.SAVE_DIR, cfg.MUTATION)
+        # if not os.path.exists(model):
+        #     # TODO: `Salesforce` part should not be hardcoded / should be configurable so that we can download models 
+        #     # from other sources.
+        #     model = f'Salesforce/{cfg.MUTATION}'
+
+        tokenizer = AutoTokenizer.from_pretrained(model)
+        model = AutoModelForCausalLM.from_pretrained(model).to(cfg.DEVICE)
+
+        # TODO: Above lines may have downloaded a fresh model if it was not already present. Now, copy the model file 
+        # to the desired location if necessary.
+
+        inputs = tokenizer(prompt, return_tensors="pt").to(cfg.DEVICE)
+        sample = model.generate(**inputs, max_length=350 + len(inputs[0]),temperature=temperature,num_beams=1,do_sample=True)
+        return tokenizer.decode(sample[0][len(inputs[0]):], truncate_before_pattern=[r"\n\n^#", "^'''", "\n\n\n"])
+
+
+def replace_word_mutation(sentence):
+    if "Add" in sentence:
+        return sentence.replace("Add", "Delete")
+    elif "Delete" in sentence:
+        return sentence.replace("Delete", "Add")
+    else:
+        return sentence
+

LLMatic-main/outputs/2024-09-03/22-14-50/.hydra/config.yaml

@@ -0,0 +1,19 @@
+RANDOM_INIT_NETS: 10
+INIT_NUM_NETS: 10
+ROLL_OUTS: 4
+NUM_NETS: 10
+START_FROM_CHECKPOINT: false
+RANDOM_NETWORKS: false
+MUTATION: codegen-6B-mono
+GENERATIONS: 200
+NET_TRAINING_EPOCHS: 50
+TEMPERATURE: 0.0
+DEVICE: cuda
+NUM_PROCESSES: 15
+RAY: true
+NUM_CPUS: 15
+NUM_GPUS: 1
+NUM_GPUS_TOTAL: 4
+DIM_MAP: 2
+N_NICHES: 100
+SAVE_DIR: ./

LLMatic-main/outputs/2024-09-03/22-14-50/.hydra/hydra.yaml

@@ -0,0 +1,182 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${hydra.sweep.dir}/.submitit/%j
+    timeout_min: 2880
+    cpus_per_task: 1
+    gpus_per_node: 1
+    tasks_per_node: 1
+    mem_gb: 30
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher
+    partition: null
+    qos: null
+    comment: null
+    constraint: null
+    exclude: null
+    gres: null
+    cpus_per_gpu: null
+    gpus_per_task: null
+    mem_per_gpu: null
+    mem_per_cpu: null
+    account: null
+    signal_delay_s: 120
+    max_num_timeout: 0
+    additional_parameters: {}
+    array_parallelism: 256
+    setup: null
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params:
+      NUM_NETS: 10, 20
+      NET_TRAINING_EPOCHS: 10, 30
+      TEMPERATURE: 0.3, 0.5, 0.7
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: llmatic
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.1
+    version_base: '1.3'
+    cwd: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main\conf
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main\outputs\2024-09-03\22-14-50
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_slurm
+      hydra/output: default
+  verbose: false

LLMatic-main/outputs/2024-09-03/22-14-50/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

LLMatic-main/outputs/2024-09-03/23-04-26/.hydra/config.yaml

@@ -0,0 +1,19 @@
+RANDOM_INIT_NETS: 10
+INIT_NUM_NETS: 10
+ROLL_OUTS: 4
+NUM_NETS: 10
+START_FROM_CHECKPOINT: false
+RANDOM_NETWORKS: false
+MUTATION: codegen-6B-mono
+GENERATIONS: 200
+NET_TRAINING_EPOCHS: 50
+TEMPERATURE: 0.0
+DEVICE: cuda
+NUM_PROCESSES: 15
+RAY: true
+NUM_CPUS: 15
+NUM_GPUS: 1
+NUM_GPUS_TOTAL: 4
+DIM_MAP: 2
+N_NICHES: 100
+SAVE_DIR: ./

LLMatic-main/outputs/2024-09-03/23-04-26/.hydra/hydra.yaml

@@ -0,0 +1,182 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${hydra.sweep.dir}/.submitit/%j
+    timeout_min: 2880
+    cpus_per_task: 1
+    gpus_per_node: 1
+    tasks_per_node: 1
+    mem_gb: 30
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher
+    partition: null
+    qos: null
+    comment: null
+    constraint: null
+    exclude: null
+    gres: null
+    cpus_per_gpu: null
+    gpus_per_task: null
+    mem_per_gpu: null
+    mem_per_cpu: null
+    account: null
+    signal_delay_s: 120
+    max_num_timeout: 0
+    additional_parameters: {}
+    array_parallelism: 256
+    setup: null
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params:
+      NUM_NETS: 10, 20
+      NET_TRAINING_EPOCHS: 10, 30
+      TEMPERATURE: 0.3, 0.5, 0.7
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: llmatic
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.1
+    version_base: '1.3'
+    cwd: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main\conf
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main\outputs\2024-09-03\23-04-26
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_slurm
+      hydra/output: default
+  verbose: false

LLMatic-main/outputs/2024-09-03/23-04-26/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

LLMatic-main/outputs/2024-09-06/05-03-40/.hydra/config.yaml

@@ -0,0 +1,19 @@
+RANDOM_INIT_NETS: 10
+INIT_NUM_NETS: 10
+ROLL_OUTS: 4
+NUM_NETS: 10
+START_FROM_CHECKPOINT: false
+RANDOM_NETWORKS: false
+MUTATION: codegen-6B-mono
+GENERATIONS: 200
+NET_TRAINING_EPOCHS: 50
+TEMPERATURE: 0.0
+DEVICE: cuda
+NUM_PROCESSES: 15
+RAY: true
+NUM_CPUS: 15
+NUM_GPUS: 1
+NUM_GPUS_TOTAL: 4
+DIM_MAP: 2
+N_NICHES: 100
+SAVE_DIR: ./

LLMatic-main/outputs/2024-09-06/05-03-40/.hydra/hydra.yaml

@@ -0,0 +1,182 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${hydra.sweep.dir}/.submitit/%j
+    timeout_min: 2880
+    cpus_per_task: 1
+    gpus_per_node: 1
+    tasks_per_node: 1
+    mem_gb: 30
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher
+    partition: null
+    qos: null
+    comment: null
+    constraint: null
+    exclude: null
+    gres: null
+    cpus_per_gpu: null
+    gpus_per_task: null
+    mem_per_gpu: null
+    mem_per_cpu: null
+    account: null
+    signal_delay_s: 120
+    max_num_timeout: 0
+    additional_parameters: {}
+    array_parallelism: 256
+    setup: null
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params:
+      NUM_NETS: 10, 20
+      NET_TRAINING_EPOCHS: 10, 30
+      TEMPERATURE: 0.3, 0.5, 0.7
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: llmatic
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.1
+    version_base: '1.3'
+    cwd: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main\conf
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main\outputs\2024-09-06\05-03-40
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_slurm
+      hydra/output: default
+  verbose: false

LLMatic-main/outputs/2024-09-06/05-03-40/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

LLMatic-main/outputs/2024-09-06/06-51-07/.hydra/config.yaml

@@ -0,0 +1,19 @@
+RANDOM_INIT_NETS: 10
+INIT_NUM_NETS: 10
+ROLL_OUTS: 4
+NUM_NETS: 10
+START_FROM_CHECKPOINT: false
+RANDOM_NETWORKS: false
+MUTATION: codegen-6B-mono
+GENERATIONS: 200
+NET_TRAINING_EPOCHS: 50
+TEMPERATURE: 0.0
+DEVICE: cuda
+NUM_PROCESSES: 15
+RAY: true
+NUM_CPUS: 15
+NUM_GPUS: 1
+NUM_GPUS_TOTAL: 4
+DIM_MAP: 2
+N_NICHES: 100
+SAVE_DIR: ./

LLMatic-main/outputs/2024-09-06/06-51-07/.hydra/hydra.yaml

@@ -0,0 +1,182 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${hydra.sweep.dir}/.submitit/%j
+    timeout_min: 2880
+    cpus_per_task: 1
+    gpus_per_node: 1
+    tasks_per_node: 1
+    mem_gb: 30
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher
+    partition: null
+    qos: null
+    comment: null
+    constraint: null
+    exclude: null
+    gres: null
+    cpus_per_gpu: null
+    gpus_per_task: null
+    mem_per_gpu: null
+    mem_per_cpu: null
+    account: null
+    signal_delay_s: 120
+    max_num_timeout: 0
+    additional_parameters: {}
+    array_parallelism: 256
+    setup: null
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params:
+      NUM_NETS: 10, 20
+      NET_TRAINING_EPOCHS: 10, 30
+      TEMPERATURE: 0.3, 0.5, 0.7
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: llmatic
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.1
+    version_base: '1.3'
+    cwd: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main\conf
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: C:\Users\kjkj0\PycharmProjects\help-research-student\LLMatic-main\outputs\2024-09-06\06-51-07
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_slurm
+      hydra/output: default
+  verbose: false