* [x] [Generating Critic Scores](#generating-critic-scores)

| `temp` | temperature for sampling generation | 0.6 |

| `output_path` | Path to save generated programs | outputs/codes/ |

We can train a critic model as a classifier that predicts the test outcomes of generated samples. For each training sample, we can follow the prior processes ([generating programs](#generating-programs) and [running unit tests](running-unit-tests)) to obtain synthetic samples and their annotations of unit test outcomes. On average, we generate 20 programs per training sample (we provided some example generated programs in `data/APPS/train/`).

We created `scripts/generate_critic_scores.sh` to generate critic scores for synthetic programs. We use the same parameters as defined in [the generating program process](generating-programs) with the following additional parameters:

| **Parameters** | **Description** | **Example Values** |

|:-----------------:|:--------------------------------------------------------------------------------------------------------:|:------------------------------:|

| `critic_scores` | Enable this to run inference on critic models and obtain critic scores | N/A |

| `gt_solutions` | Enable this to run inference on ground-truth programs; else, synthetic programs are used by default | N/A |

Other parameters are defined in the file `utils/generate_configs.py`.

Running the generation script will output programs, each of which is saved into a `pkl` (pickle) file, including data fields `code` (list of programs), `prompt` (constructed input sequence to the critic model), `gt_error_type` (ground-truth test outcomes), `pred_error_type` (predicted test outcomes by critic), `error_hidden_states` (hidden states returned by critic).

parser.add_argument("--critic_scores", default=False, action='store_true', help='if model is a critic model, enable this to output critic scores')

parser.add_argument('--gt_solutions', default=False, action='store_true', help='Only when critic is used, enable this to estimate returns/rewards for ground-truth programs, else synthetic programs by default')

import pickle as pkl

import numpy as np

from collections import Counter

import datasets.utils as dsutils

def generate_critic_inputs(args, test_case_path, prompt_path, solutions_path, tokenizer,

starter_path=None, gt_solutions=False):

_input = generate_prompt(args, test_case_path, prompt_path, solutions_path, tokenizer, starter_path)

q_tokens = tokenizer.encode(_input, verbose=False, max_length=args.source_len)

in_tokens = [tokenizer.eos_token_id] * args.source_len

in_tokens[:len(q_tokens)] = q_tokens

in_tokens = in_tokens[:args.source_len]

solutions = json.load(open(solutions_path, 'r'))

all_texts = []

gt_errors = []

all_codes = []

for sol_index, solution in enumerate(solutions):

if gt_solutions:

solution_str = dsutils.reindent_code(solution)

else:

solution_str = dsutils.reindent_code(solution['code'])

a_tokens = tokenizer.encode(solution_str)

code = [-100] * args.max_len

code[:len(a_tokens)] = a_tokens

code = code[:args.max_len]

all_texts.append(in_tokens)

all_codes.append(code)

if gt_solutions:

gt_errors.append(dsutils.get_error_type(True))

else:

gt_errors.append(dsutils.get_error_type(solution['result']))

return all_texts, all_codes, gt_errors

if args.critic_scores:

model = T5ForConditionalGeneration.from_pretrained(args.model_path, tuning_mode='critic')

else:

model = T5ForConditionalGeneration.from_pretrained(args.model_path)

if args.critic_scores:

all_preds = []

all_gts = []

if args.critic_scores and \

os.path.exists(os.path.join(args.output_path, f"{problem_id}_gt{args.gt_solutions}.pkl")):

elif os.path.exists(os.path.join(args.output_path, f"{problem_id}.json")):

continue

if args.critic_scores and not args.gt_solutions:

solutions_path = os.path.join(prob_path, "gen_solutions.json")

else:

solutions_path = os.path.join(prob_path, "solutions.json")

if args.critic_scores:

input_texts, input_codes, gt_error_types = generate_critic_inputs(args, test_case_path, prompt_path, solutions_path,

tokenizer, starter_path, args.gt_solutions)

else:

input_text = generate_prompt(args, test_case_path, prompt_path, solutions_path,

if args.critic_scores:

text_tensor = torch.tensor(input_texts).to(device)

code_tensor = torch.tensor(input_codes).to(device)

gt_error_tensor = torch.tensor(gt_error_types).to(device)

curr_inputs = {'input_ids': text_tensor, 'error_types': gt_error_tensor, 'labels': code_tensor}

_, error_preds, error_hidden_states = model(**curr_inputs, return_error_hidden_states=True)

assert len(gt_error_types) == len(error_preds)

all_preds.extend(error_preds.cpu().numpy().tolist())

all_gts.extend(gt_error_types)

saved_critic_scores = {}

saved_critic_scores[problem_id] = {'code': input_codes, 'prompt': input_texts,

'gt_error_type': gt_error_types,

'pred_error_type': error_preds.cpu().numpy(),

'error_hidden_states': error_hidden_states.cpu().numpy()}

scores_loc = os.path.join(args.output_path, f"{problem_id}_gt{args.gt_solutions}.pkl")

pkl.dump(saved_critic_scores, open(scores_loc, 'wb'))

else:

input_ids = torch.LongTensor(tokenizer.encode(input_text,

verbose=False,

max_length=args.source_len)).unsqueeze(0).cuda()

num_loops = int(args.num_seqs / args.num_seqs_per_iter)

output_programs = []

for i in tqdm(range(num_loops), ncols=0, total=num_loops, leave=False):

output_ids = model.generate(

input_ids,

do_sample=True,

temperature=args.temperature,

max_length=args.max_len,

num_return_sequences=args.num_seqs_per_iter,

top_p=0.95)

for output_id in output_ids:

output_programs.append(tokenizer.decode(output_id, skip_special_tokens=True))

saved_codes = {}

saved_codes[problem_id] = {'code': output_programs, 'prompt': input_text}

codes_loc = os.path.join(args.output_path, f"{problem_id}.json")

with open(codes_loc, "w") as f:

json.dump(saved_codes, f)

if args.critic_scores:

print("Total number of samples: {}".format(len(all_gts)))

acc = (np.array(all_preds) == np.array(all_gts)).sum()/len(all_gts)

print("Error Pred Acc: {}".format(acc))

print("Prediction distribution: {}".format(Counter(all_preds)))

print("GT distribution: {}".format(Counter(all_gts)))

critic_path=models/codet5_finetuned_critic/

tokenizer_path=models/codet5_tokenizer/

test_path=data/APPS/train/ #test.json

output_path=outputs/critic_scores/

CUDA_VISIBLE_DEVICES=0 python generate.py \

--model_path ${critic_path} \

--test_path ${test_path} \

--output_path ${output_path} \

--critic_scores --gt_solutions

return_error_hidden_states=False

if return_error_hidden_states:

return error_pred_loss, error_preds, error_states