We now support GPT 5.6

.gitignore

@@ -1 +1,33 @@
-__pycache__
+__pycache__
+*.pyc
+
+# --- benchmark run artifacts (not part of the framework) ---
+# Individual per-row outputs; only the merged submission file is kept.
+examples/travel planner/plan*.json
+!examples/travel planner/merged_plans.jsonl
+
+# Archived / backup submissions and pre-fix snapshots
+examples/travel planner/merged_plans.*.jsonl
+examples/travel planner/pre_transport_fix/
+examples/travel planner/pre_badcase_fix/
+examples/travel planner/old_plans_yunwu/
+
+# Run logs, pids, editor temp files
+examples/travel planner/*.log
+examples/travel planner/*.pid
+examples/travel planner/run_logs/
+*.tmp.*
+log.txt
+logs/
+files/
+
+# --- evaluation harness / internal QC (not framework code) ---
+examples/travel planner/run_parallel.py
+examples/travel planner/check_plans.py
+examples/travel planner/salvage_loop.py
+
+# Partial benchmark outputs from the other examples (not published)
+examples/GSM8k/plan*.json
+examples/MATH/plan*.json
+examples/humaneval/plan*.json
+examples/mbpp/plan*.json

README.md

@@ -18,6 +18,30 @@ To run the latest version, you can add your key and change the prompt in `config
 
 Some experiments are shown in `examples/` using an older version of MegaAgent. You can use the same prompt while substituting other files with the latest version.
 
+### Backbone / API interface
+
+The main MegaAgent code (`.`) and the TravelPlanner example talk to the model
+through the modern OpenAI Python SDK tool-use interface (`tools` /
+`tool_calls` / `role:"tool"`). The shared transport lives in a single file,
+`llm_core.py`, which their `llm.py` delegates to; all framework logic, prompts,
+and tool schemas are unchanged from the original design. Configure the backbone
+in `config.py`:
+
+```python
+api_key = 'YOUR_KEY'
+model = "gpt-5.6-sol"
+base_url = 'https://your-endpoint/v1'
+reasoning_effort = 'xhigh'   # optional; sent only when set
+```
+
+`llm_core.py` streams every request internally (reassembling one complete
+response), sends no `temperature` and never caps `max_tokens` (so long
+reasoning is never truncated), and sanitizes histories to the strict tool-use
+protocol. Install dependencies with `pip install -r requirements.txt`.
+
+The other examples under `examples/` still use the legacy
+`functions`/`function_call` interface with `url` in their `config.py`.
+
 ## Experimental Results
 
 ### RQ1: Quantitative experiments using gpt-4o as backbone
@@ -42,14 +66,21 @@ Some experiments are shown in `examples/` using an older version of MegaAgent. Y
 
 
 
-We also used GPT-4o to achieve the following results on TravelPlanner. The submission file is included in `examples/travel planner`.
+We also evaluated MegaAgent on TravelPlanner (validation set, sole-planning
+mode). The submission file (`merged_plans.jsonl`) is included in
+`examples/travel planner`.
+
+| Metric | GPT-4o | GPT-5.6 |
+| ------ | ------ | ------- |
+| Delivery Rate | 100.0% | 100.0% |
+| Commonsense Constraint Micro Pass Rate | 81.88% | 97.64% |
+| Commonsense Constraint Macro Pass Rate | 27.22% | 84.44% |
+| Hard Constraint Micro Pass Rate | 40.48% | 87.14% |
+| Hard Constraint Macro Pass Rate | 23.89% | 83.33% |
+| **Final Pass Rate** | **10.0%** | **76.67%** |
 
-- Delivery Rate: 100.0%
-- Commonsense Constraint Micro Pass Rate: 81.88%
-- Commonsense Constraint Macro Pass Rate: 27.22%
-- Hard Constraint Micro Pass Rate: 40.48%
-- Hard Constraint Macro Pass Rate: 23.89%
-- Final Pass Rate: 10.0%
+The GPT-5.6 column uses `gpt-5.6-sol` with `reasoning_effort=xhigh` through the
+tool-use interface described above.
 
 
 ## Licenses

agent.py

@@ -18,7 +18,7 @@ def __init__(self, agent_name, initial_message):
         self.initialize_logger(agent_name)
 
     def add_memory(self, memory):
-        if (memory['role']!='function' and memory['content'] != None):
+        if (memory['role'] not in ('function', 'tool') and memory['content'] != None):
             self.history_pool.add(documents=[memory['content']], ids=[str(time.time())])
         self.logger.info(str(memory))
         self.history.append(memory)
@@ -75,7 +75,9 @@ def get(self):
 
         if self.history and self.history[-1]['content']:
             relevant_history = self.history_pool.query(query_texts=self.history[-1]['content'], n_results=1)
-            if relevant_history:
+            # chroma returns empty result lists on a fresh collection; the
+            # bare [0][0] index would kill this agent's worker thread
+            if relevant_history and relevant_history['documents'] and relevant_history['documents'][0]:
                 init+=f"\n\nHere is a relevant memory: \n{relevant_history['documents'][0][0]}\nBelow is the recent dialogue."
 
         memory = [{"role": "system", "content": init}]
@@ -257,7 +259,7 @@ def run(self):
                 req = self.get()
                 if llm_output != None:
                     self.logger.info(f"Assistant: {llm_output}")
-                if 'function_call' not in assistant_output:
+                if not assistant_output.get('tool_calls'):
                     self.add_dialogue("user", "Error: No function call found in the response. You must use function calls to work and communicate with other agents. If you have nothing to do now, please call 'terminate' function.")
                     req = self.get()
                     round += 1
@@ -266,21 +268,25 @@ def run(self):
 
             round = 0
             while round < config.MAX_ROUNDS:
-                tool_call = assistant_output['function_call']
-                tool_name = tool_call['name']
-                arguments = json.loads(tool_call['arguments'])
-                tool_info = self.execute(tool_name, {"role": "function"}, arguments)
-                if tool_info == {}:
+                terminated = False
+                for tool_call in assistant_output.get('tool_calls', []):
+                    tool_name = tool_call['function']['name']
+                    arguments = json.loads(tool_call['function']['arguments'])
+                    tool_info = self.execute(tool_name, {"role": "tool", "tool_call_id": tool_call['id']}, arguments)
+                    if tool_info == {}:
+                        terminated = True
+                        break
+                    self.add_memory(tool_info)
+                    req += [tool_info]
+                if terminated:
                     break
-                self.add_memory(tool_info)
-                req += [tool_info]
                 round += 1
                 response = get_llm_response(req, agent_name=self.name)
                 assistant_output = response['choices'][0]['message']
                 llm_output = assistant_output['content']
                 self.add_memory(assistant_output)
                 req += [assistant_output]
-                while 'function_call' not in assistant_output:
+                while not assistant_output.get('tool_calls'):
                     req += [{"role":"user", "content": "Error: No function call found in the response. You must use function calls to work and communicate with other agents. If you have nothing to do now, please call 'terminate' function."}]
                     response = get_llm_response(req, agent_name=self.name)
                     assistant_output = response['choices'][0]['message']

config.py

@@ -1,6 +1,7 @@
 api_key = 'sk-your_api_key_here'
-model = "gpt-4.1"
-url = 'https://api.openai.com/v1/chat/completions'
+model = "gpt-5.6-sol"
+base_url = 'https://api.openai.com/v1'
+reasoning_effort = 'xhigh'
 
 MAX_MEMORY = 10
 MAX_ROUNDS = 20

examples/travel planner/config.py

@@ -1,6 +1,7 @@
 api_key = 'sk-'
-model = "gpt-4o"
-url = 'https://api.openai.com/v1/chat/completions'
+model = "gpt-5.6-sol"
+base_url = 'https://api.openai.com/v1'
+reasoning_effort = 'xhigh'
 
 MAX_LEN = 6
 MAX_ROUNDS = 15

examples/travel planner/execute.py

@@ -1,23 +1,21 @@
-import pandas as pd
+import argparse
+import json
 import os
 import shutil
 import subprocess
+import sys
+import time
+
+import pandas as pd
 
-# 文件路径
+# file paths
 val_file_path = 'travel_planner_val.xlsx'
 config_file_path = 'config.py'
 main_script_path = 'main.py'
 output_folder = './'
 plan_folder = 'files/plan.json'
 
-# 加载 Excel 文件
-val_data = pd.read_excel(val_file_path)
-
-# 读取 config.py 文件
-with open(config_file_path, 'r', encoding='utf-8') as file:
-    config_content = file.read()
-
-# 修改 additional_prompt 的函数
+# rewrite additional_prompt for one row
 def update_additional_prompt(config_content, row):
     query = row['query']
     ref_info = row['reference_information']
@@ -28,7 +26,7 @@ def update_additional_prompt(config_content, row):
 where "-" denotes not applicable(like the accommodation of the last day, or the meal on the plane/car).
 '''+f'''
 Here are the customers' requirements:
-{query} You cannot choose the same restaurant for two different meals.
+{query} You cannot choose the same restaurant for two different meals. Keep the transportation mode consistent across the whole trip: if you take a flight on any day, do not use self-driving on any day (you cannot fly and drive your own car in the same trip), and vice versa.
 
 Here are all the needed information. You cannot query more. Be careful with room rules and Minimum Nights Stay!
 {ref_info}
@@ -49,29 +47,133 @@ def update_additional_prompt(config_content, row):
     )
     return updated_content
 
-# 针对每一行进行处理
-for index, row in val_data.iterrows():
-    # 修改 config.py 文件
+
+def plan_file_is_valid(path):
+    """Minimal structural check used for skip/salvage decisions."""
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return isinstance(data.get('plan'), list) and len(data['plan']) > 0
+    except Exception:
+        return False
+
+
+def kill_process_tree(proc):
+    subprocess.run(['taskkill', '/F', '/T', '/PID', str(proc.pid)],
+                   capture_output=True)
+
+
+def run_row(index, row, config_content, timeout):
+    """Run main.py for one benchmark row. Returns a status string."""
     updated_config_content = update_additional_prompt(config_content, row)
-
-    # 临时保存更新后的 config.py
-    temp_config_path = f'config.py'
-    with open(temp_config_path, 'w', encoding='utf-8') as file:
-        file.write(updated_config_content)
-
-    # 运行 main.py
+    with open(config_file_path, 'w', encoding='utf-8') as f:
+        f.write(updated_config_content)
+
+    # main.py removes log.txt unconditionally at startup
+    if not os.path.exists('log.txt'):
+        with open('log.txt', 'w', encoding='utf-8') as f:
+            f.write('')
+
+    env = dict(os.environ)
+    env['PYTHONUTF8'] = '1'
+    os.makedirs('run_logs', exist_ok=True)
+    console_path = os.path.join('run_logs', f'console_row{index}.txt')
+
+    status = 'ok'
+    started = time.time()
+    with open(console_path, 'w', encoding='utf-8', errors='replace') as console:
+        proc = subprocess.Popen([sys.executable, main_script_path],
+                                stdout=console, stderr=subprocess.STDOUT,
+                                env=env)
+        try:
+            rc = proc.wait(timeout=timeout)
+            if rc != 0:
+                status = f'exit code {rc}'
+        except subprocess.TimeoutExpired:
+            kill_process_tree(proc)
+            proc.wait()
+            status = f'timeout after {timeout}s'
+    elapsed = time.time() - started
+
+    # Collect the produced plan (also salvages timeout/crash runs whose
+    # plan.json was already written)
+    copied = False
+    if os.path.exists(plan_folder) and plan_file_is_valid(plan_folder):
+        output_plan_path = os.path.join(output_folder, f'plan{index}.json')
+        shutil.copy(plan_folder, output_plan_path)
+        copied = True
+
+    # Archive the run log for post-mortems
     try:
-        subprocess.run(["python", main_script_path], check=True)
-
-        # 复制生成的 plan.json 到目标文件
-        if os.path.exists(plan_folder):
-            output_plan_path = os.path.join(output_folder, f'plan{index}.json')
-            shutil.copy(plan_folder, output_plan_path)
-            print(f"Plan for row {index} saved as {output_plan_path}.")
-        else:
-            print(f"Plan for row {index} not found. Ensure main.py created the file.")
-    except subprocess.CalledProcessError as e:
-        print(f"Error running main.py for row {index}: {e}")
-    except FileNotFoundError:
-        print(f"File not found during processing of row {index}. Ensure paths are correct.")
-
+        if os.path.exists('log.txt'):
+            shutil.copy('log.txt', os.path.join('run_logs', f'log_row{index}.txt'))
+    except Exception:
+        pass
+
+    if copied and status != 'ok':
+        status += ' (plan salvaged)'
+    elif not copied:
+        status += '; no valid plan produced' if status != 'ok' else 'no valid plan produced'
+    print(f"Row {index}: {status} [{elapsed/60:.1f} min]", flush=True)
+    return copied
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='TravelPlanner benchmark runner')
+    parser.add_argument('--start', type=int, default=0, help='first row index (inclusive)')
+    parser.add_argument('--end', type=int, default=None, help='last row index (exclusive)')
+    parser.add_argument('--only', type=str, default=None,
+                        help='comma-separated row indices to (re)run, overrides --start/--end')
+    parser.add_argument('--timeout', type=int, default=10800,
+                        help='per-row timeout in seconds (default 3h)')
+    parser.add_argument('--force', action='store_true',
+                        help='rerun rows even if a valid plan{i}.json exists')
+    parser.add_argument('--retries', type=int, default=2,
+                        help='extra passes over rows that still have no valid plan')
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    # load the validation set
+    val_data = pd.read_excel(val_file_path)
+
+    # read config.py once (the anchors are stable, so rewriting is idempotent)
+    with open(config_file_path, 'r', encoding='utf-8') as file:
+        config_content = file.read()
+
+    if args.only:
+        indices = [int(x) for x in args.only.split(',') if x.strip() != '']
+    else:
+        end = len(val_data) if args.end is None else min(args.end, len(val_data))
+        indices = list(range(args.start, end))
+
+    for attempt in range(1 + max(0, args.retries)):
+        pending = []
+        for index in indices:
+            plan_path = os.path.join(output_folder, f'plan{index}.json')
+            if not args.force and plan_file_is_valid(plan_path):
+                continue
+            pending.append(index)
+        if not pending:
+            break
+        if attempt > 0:
+            print(f"Retry pass {attempt}: {len(pending)} rows still missing "
+                  f"valid plans: {pending}", flush=True)
+        for index in pending:
+            run_row(index, val_data.iloc[index], config_content, args.timeout)
+        args.force = False  # retries only target still-invalid rows
+
+    missing = [i for i in indices
+               if not plan_file_is_valid(os.path.join(output_folder, f'plan{i}.json'))]
+    done = len(indices) - len(missing)
+    print(f"\nDone: {done}/{len(indices)} rows have valid plans.", flush=True)
+    if missing:
+        print(f"Still missing: {missing}", flush=True)
+        print(f"Rerun with: python execute.py --only "
+              f"{','.join(str(i) for i in missing)}", flush=True)
+
+
+if __name__ == '__main__':
+    main()