Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,38 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/Evaluation/PAL-Math/datasets/tabmwp/test.json filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/numpy/_core/_multiarray_umath.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pandas/_libs/algos.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pandas/_libs/groupby.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pandas/_libs/hashtable.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pandas/_libs/interval.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pandas/_libs/join.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pandas/_libs/tslibs/offsets.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/_compute.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/_flight.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/lib.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/libarrow.1900.dylib filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/libarrow_acero.1900.dylib filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/libarrow_dataset.1900.dylib filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/libarrow_flight.1900.dylib filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/libarrow_python.1900.0.0.dylib filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/libarrow_python.1900.dylib filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/libarrow_python.dylib filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/libarrow_substrait.1900.dylib filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pyarrow/libparquet.1900.dylib filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/pydantic_core/_pydantic_core.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/sympy/polys/benchmarks/__pycache__/bench_solvers.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/tokenizers/tokenizers.abi3.so filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/torch/bin/protoc filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/torch/bin/protoc-3.13.0.0 filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/torch/lib/libtorch_cpu.dylib filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/torch/lib/libtorch_python.dylib filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/wandb/bin/gpu_stats filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/finetune/venv/lib/python3.12/site-packages/wandb/bin/wandb-core filter=lfs diff=lfs merge=lfs -text
+DeepSeek-Coder-main/pictures/home.tif filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.12/site-packages/numpy/_core/_multiarray_umath.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.12/site-packages/pydantic_core/_pydantic_core.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.12/site-packages/tiktoken/_tiktoken.cpython-312-darwin.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.12/site-packages/wandb/bin/gpu_stats filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.12/site-packages/wandb/bin/wandb-core filter=lfs diff=lfs merge=lfs -text

DeepSeek-Coder-main/.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+Evaluation/MBPP/eval_instruct.sh
+Evaluation/LeetCode/output/

DeepSeek-Coder-main/Evaluation/DS-1000/README.md

@@ -0,0 +1,29 @@
+## 1. Introduction
+
+We provide a test script to evaluate the performance of the **deepseek-coder** model on code completion benchmarks. We select the widely-used benchmarks: [**DS-1000**](https://github.com/xlang-ai/DS-1000).
+
+## 2. Evaluation
+
+We directly use the scripts provided by the DS-1000 repository to evaluate the performance of the models. You can refer to [**DS-1000**](https://github.com/xlang-ai/DS-1000) to find more details about the evaluation.
+
+
+## 3. Experimental Results
+
+We report experimental results here for the completion mode of DS-1000. We set the maximum length to **2048**, and employ the **greedy search strategy**.  To ensure a fair comparison, we apply identical hyper-parameters across all open-source models under evaluation.
+
+| Model                  | Size | Matplotlib | Numpy | Pandas | Pytorch | Scipy | Scikit-Learn | Tensorflow | Avg   |
+|------------------------|------|------------|-------|--------|---------|-------|-------------|------------|-------|
+| Codex-001              | -    | 41.8%      | 26.6% | 9.4%   | 9.7%    | 15.0% | 18.5%        | 17.2%      | 20.2% |
+| Codex-002              | -    | **57.0%**      | 43.1% | **26.5%**  | **41.8%**   | 31.8% | **44.8%**        | 39.3%      | 39.2% |
+| CodeShell              | 7B   | 34.1%      | 21.8% | 10.7%  | 11.8%   | 17.0% | 20.0%        | 15.6%      | 18.8% |
+| CodeGeeX2              | 6B   | 38.7%      | 26.8% | 14.4%  | 11.8%   | 19.8% | 27.0%        | 17.8%      | 22.9% |
+| StarCoder         | 16B  | 47.7%      | 31.4% | 12.7%  | 25%   | 22.6% | 35.7%        | 22.2%      | 27.2% |
+| CodeLLama-Base         | 7B   | 41.9%      | 24.6% | 14.8%  | 16.2%   | 18.9% | 17.4%        | 17.8%      | 22.1% |
+| CodeLLama-Base         | 13B  | 46.5%      | 28.6% | 18.2%  | 19.1%   | 18.9% | 27.8%        | 33.3%      | 26.8% |
+| CodeLLama-Base         | 34B  | 50.3%      | 42.7% | 23.0%  | 25.0%   | 28.3% | 33.9%        | 40.0%      | 34.3% |
+| | | | |  |  |  |  |  |  | |
+| DeepSeek-Coder-Base    | 1.3B   | 32.3%      | 21.4% | 9.3%   | 8.8%    | 8.5%  | 16.5%        | 8.9%       | 16.2% |
+| DeepSeek-Coder-Base    | 5.7B   | 51.1%      | 31.8% | 19.9%  | 14.7%   | 17.0% | 29.6%        | 15.6%      | 27.7% |
+| DeepSeek-Coder-Base    | 6.7B   | 48.4%      | 35.5% | 20.6%  | 19.1%   | 22.6% | 38.3%        | 24.4%      | 30.5% |
+| DeepSeek-Coder-Base    | 33B  | 56.1%      | **49.6%** | 25.8%  | 36.8%   | **36.8%** | 40.0%        | **46.7%**      | **40.2%** |
+

DeepSeek-Coder-main/Evaluation/HumanEval/README.md

@@ -0,0 +1,74 @@
+## 1. Introduction
+
+We provide a test script to evaluate the performance of the **deepseek-coder** model on code generation benchmarks. We select the widely-used benchmarks: **[HumanEval-Python](https://huggingface.co/datasets/openai_humaneval), [HumanEval-Multilingual](https://huggingface.co/datasets/nuprl/MultiPL-E)**.
+
+
+
+## 2. Setup
+
+```
+pip install accelerate
+pip install attrdict
+pip install transformers
+pip install pytorch
+```
+
+
+## 3. Evaluation
+
+We've created a sample script, **eval.sh**, that demonstrates how to test the **DeepSeek-Coder-1.3b-Base** model on the HumanEval dataset leveraging **8** GPUs. If your use case involves a different model or dataset, simply adjust the script to fit your needs.
+
+Additionally, for various programming languages, the execution path may differ. Please ensure you update the appropriate paths in the **humaneval/execution.py** file accordingly.
+
+```bash
+MODEL_NAME_OR_PATH="deepseek-ai/deepseek-coder-1.3b-base"
+DATASET_ROOT="data/"
+LANGUAGE="python"
+python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT} 
+```
+
+To evaluate the instruction-based model, please follow the script below:
+```bash
+LANG="python"
+OUPUT_DIR="output"
+MODEL="deepseek-coder-33b-instruct"
+
+CUDA_VISIBLE_DEVICES=0,1 python eval_instruct.py \
+    --model "deepseek-ai/$MODEL" \
+    --output_path "$OUPUT_DIR/${LANG}.$MODEL.jsonl" \
+    --language $LANG \
+    --temp_dir $OUPUT_DIR
+```
+
+## 4. Experimental Results
+
+We report experimental results here for 8 main-stream programming languages, **python**, **c++**, **java**, **PHP**, **TypeScript**, **C#**, **Bash**, and **JavaScript**. For all open-source models, we utilize this repository to obtain the performance of the models on the HumanEval dataset. We set the maximum input length to **4096** and the maximum output length to **500**, and employ the **greedy search strategy**.
+
+
+#### (1) Multilingual Base Models
+
+| Model             | Size | Python | C++   | Java | PHP  | TS   | C#   | Bash | JS   | Avg  |
+|-------------------|------|--------|-------|------|------|------|------|------|------|------|
+| code-cushman-001  | 12B  | 33.5%  | 31.9% | 30.6%| 28.9%| 31.3%| 22.1%| 11.7%| -    | -    |
+| CodeShell         | 7B   | 35.4%  | 32.9% | 34.2%| 31.7%| 30.2%| 38.0%| 7.0% | 33.5%| 30.4%|
+| CodeGeeX2         | 6B   | 36.0%  | 29.2% | 25.9%| 23.6%| 20.8%| 29.7%| 6.3% | 24.8%| 24.5%|
+| StarCoderBase     | 16B  | 31.7%  | 31.1% | 28.5%| 25.4%| 34.0%| 34.8%| 8.9% | 29.8%| 28.0%|
+| CodeLLama         | 7B   | 31.7%  | 29.8% | 34.2%| 23.6%| 36.5%| 36.7%| 12.0%| 29.2%| 29.2%|
+| CodeLLama         | 13B  | 36.0%  | 37.9% | 38.0%| 34.2%| 45.2%| 43.0%| 16.5%| 32.3%| 35.4%|
+| CodeLLama         | 34B  | 48.2%  | 44.7% | 44.9%| 41.0%| 42.1%| 48.7%| 15.8%| 42.2%| 41.0%|
+| | | | |  |  |  |  |  |  | |
+| DeepSeek-Coder-Base| 1.3B   | 34.8%  | 31.1% | 32.3%| 24.2%| 28.9%| 36.7%| 10.1%| 28.6%| 28.3%|
+| DeepSeek-Coder-Base| 5.7B   | 48.7%  | 45.3% | 41.1%| 39.7%| 44.7%| 41.1%| 27.8%| 42.2%| 41.3%|
+| DeepSeek-Coder-Base| 6.7B   | 49.4%  | 50.3% | 43.0%| 38.5%| 49.7%| 50.0%| 28.5%| 48.4%| 44.7%|
+| DeepSeek-Coder-Base|33B  | **56.1%**  | **58.4%** | **51.9%**| **44.1%**| **52.8%**| **51.3%**| **32.3%**| **55.3%**| **50.3%**|
+
+#### (2) Instruction-Tuned Models
+| Model               | Size | Python | C++   | Java | PHP  | TS   | C#   | Bash | JS   | Avg  |
+|---------------------|------|--------|-------|------|------|------|------|------|------|------|
+| GPT-3.5-Turbo         | -    | 76.2%  | 63.4% | 69.2%| 60.9%| 69.1%| 70.8%| 42.4%| 67.1%| 64.9%|
+| GPT-4               | -    | **84.1%**  | **76.4%** | **81.6%**| **77.2%**| **77.4%**| **79.1%**| **58.2%**| **78.0%**| **76.5%**|
+| | | | |  |  |  |  |  |  | |
+| DeepSeek-Coder-Instruct | 1.3B  | 65.2%      | 45.3%    | 51.9%    | 45.3%    | 59.7%   |55.1%    | 12.7%    | 52.2%    | 48.4%    |
+| DeepSeek-Coder-Instruct | 6.7B  | 78.9%  | 63.4% | 68.4% | 68.9%| 67.2%| 72.8%| 36.7%| 72.7%| 66.1%|
+| DeepSeek-Coder-Instruct | 33B | **79.3%**  | **68.9%** | **73.4%** | **72.7%**| **67.9%**| **74.1%**| **43.0%**| **73.9%**| **69.2%**|
+

DeepSeek-Coder-main/Evaluation/HumanEval/eval.sh

@@ -0,0 +1,4 @@
+MODEL_NAME_OR_PATH="deepseek/deepseek-coder-1b"
+DATASET_ROOT="data/"
+LANGUAGE="python"
+CUDA_VISIBLE_DEVICES=1,2,3 python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT}

DeepSeek-Coder-main/Evaluation/HumanEval/eval_instruct.py

@@ -0,0 +1,129 @@
+import argparse
+import json
+import os
+import torch
+from pathlib import Path
+from tqdm import tqdm
+
+data_abs_dir = Path(__file__).parent / "data"
+
+from utils.utils import extract_generation_code, languge_settings
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from human_eval.evaluation import evaluate_functional_correctness
+
+def build_deepseekcoder_instruction(languge: str, question: str):
+    return '''
+Please continue to complete the function. You are not allowed to modify the given code and do the completion only. Please return all completed function in a codeblock. Here is the given code to do completion:
+```{}
+{}
+```
+'''.strip().format(languge.lower(), question.strip())
+
+def generate_one(example, lang, tokenizer, model):
+    prompt = build_deepseekcoder_instruction(languge_settings[lang]['full_name'], example['prompt'])
+    inputs = tokenizer.apply_chat_template(
+        [{'role': 'user', 'content': prompt }],
+        return_tensors="pt",
+        add_generation_prompt=True
+    ).to(model.device)
+
+    stop_id = tokenizer.convert_tokens_to_ids("<|EOT|>")
+    assert isinstance(stop_id, int), "Invalid tokenizer, EOT id not found"
+
+    outputs = model.generate(
+        inputs, 
+        max_new_tokens=1024,
+        do_sample=False,
+        # top_p=0.95,
+        # temperature=temperature,
+        pad_token_id=stop_id,
+        eos_token_id=stop_id
+    )
+
+    output = tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True)
+    example['output'] = output
+    
+    return extract_generation_code(example, lang_code=lang)
+
+def generate_main(args):
+    model_name_or_path = args.model
+    lang = args.language
+    saved_path = args.output_path
+    temp_dir = args.temp_dir
+    os.makedirs(temp_dir, exist_ok=True)
+    problem_file = os.path.join(data_abs_dir, f"humaneval-{lang}.jsonl")
+
+    print("model", model_name_or_path)
+    tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
+    print("load tokenizer {} from {} over.".format(tokenizer.__class__, model_name_or_path))
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name_or_path,
+        torch_dtype=torch.bfloat16,
+        device_map="auto",
+        #use_flash_attention_2=True
+    )
+    model.eval()
+    examples = [json.loads(x) for x in open(problem_file) if x.strip()]
+    print("Read {} examples for evaluation over.".format(len(examples)))
+
+    generated_examples = []
+    for ex in tqdm(examples, desc='Generating'):
+        gen_example = generate_one(ex, args.language, tokenizer, model)
+        generated_examples.append(gen_example)
+
+    print("Generate all over!!!")
+    with open(saved_path, 'w', encoding='utf-8') as fw:
+        for ex in generated_examples:
+            fw.write(json.dumps(ex) + '\n')
+        print("Save {} processed examples into {} over!".format(len(generated_examples), saved_path))
+    
+    result = evaluate_functional_correctness(
+        input_file=saved_path,
+        tmp_dir=temp_dir,
+        n_workers=8,
+        timeout=3.0,
+        problem_file=problem_file,
+        language=lang
+    )
+    print(lang, result, model_name_or_path)
+    pass
+
+def evaluation_only(args):
+    lang = args.language
+    temp_dir = args.temp_dir
+    assert os.path.exists(args.output_path), "Not fond output file: {}".format(args.output_path)
+    os.makedirs(temp_dir, exist_ok=True)
+
+    output_name = os.path.basename(args.output_path)
+    output_examples = [json.loads(x) for x in open(args.output_path) if x.strip()]
+
+    processed_examples = [extract_generation_code(ex, lang) for ex in tqdm(output_examples, "Processing")]
+    processed_path = os.path.join(temp_dir, output_name)
+    with open(processed_path, 'w', encoding='utf-8') as fw:
+        for ex in processed_examples:
+            fw.write(json.dumps(ex) + '\n')
+        print("Save {} processed examples into {} over!".format(len(processed_examples), processed_path))
+
+    problem_file = os.path.join(data_abs_dir, f"humaneval-{lang}.jsonl")
+    from human_eval.evaluation import evaluate_functional_correctness
+    result = evaluate_functional_correctness(
+        input_file=processed_path,
+        tmp_dir=temp_dir,
+        n_workers=8,
+        timeout=3.0,
+        problem_file=problem_file,
+        language=lang
+    )
+    print(lang, result)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--model', type=str, help="model name or path")
+    parser.add_argument('--output_path', type=str, help="output path of your generation")
+    parser.add_argument('--language', type=str, help="langauge")
+    parser.add_argument('--temp_dir', type=str, help="temp dir for evaluation", default="tmp")
+    args = parser.parse_args()
+
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+    generate_main(args)
+    pass

DeepSeek-Coder-main/Evaluation/HumanEval/eval_pal.py

@@ -0,0 +1,42 @@
+import os
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn.functional as F
+import json
+import torch.distributed as dist
+import subprocess
+import sys
+from accelerate import Accelerator
+from accelerate import DistributedDataParallelKwargs
+from pathlib import Path
+from argparse import ArgumentParser
+from humaneval import HumanEval as evaltor
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+if __name__ == '__main__':
+    kwargs_handlers = [DistributedDataParallelKwargs(find_unused_parameters=True)]
+    accelerator = Accelerator(mixed_precision="bf16", kwargs_handlers=kwargs_handlers)   
+
+
+    parser = ArgumentParser()
+    parser.add_argument("--logdir", type=str, default="")
+    parser.add_argument("--language", type=str, default="")
+    parser.add_argument("--dataroot", type=str, default="")
+    args = parser.parse_args()
+
+    logdir = args.logdir
+    language = args.language
+
+    if logdir == "":
+        logdir = "tmp/"
+    tokenizer = dict(
+        cls=AutoTokenizer,
+        model_path=logdir,)
+
+    dataroot = args.dataroot
+
+    evaluator = evaltor(data_root=dataroot, max_seq_len=4096, tokenizer_cfg=tokenizer, log_dir=logdir, n_sample=1, batch_size=1, language=language, max_gen_len=500)
+    model = AutoModelForCausalLM.from_pretrained(logdir, device_map=accelerator.device, trust_remote_code=True, torch_dtype=torch.bfloat16)
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+    evaluator.eval_model(model, accelerator)

DeepSeek-Coder-main/Evaluation/HumanEval/human_eval/data.py

@@ -0,0 +1,49 @@
+from typing import Iterable, Dict
+import gzip
+import json
+import os
+
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+HUMAN_EVAL = os.path.join(ROOT, "..", "data", "HumanEval.jsonl.gz")
+
+
+def read_problems(evalset_file: str = HUMAN_EVAL) -> Dict[str, Dict]:
+    return {task["task_id"]: task for task in stream_jsonl(evalset_file)}
+
+
+def stream_jsonl(filename: str) -> Iterable[Dict]:
+    """
+    Parses each jsonl line and yields it as a dictionary
+    """
+    if filename.endswith(".gz"):
+        with open(filename, "rb") as gzfp:
+            with gzip.open(gzfp, 'rt') as fp:
+                for line in fp:
+                    if any(not x.isspace() for x in line):
+                        yield json.loads(line)
+    else:
+        with open(filename, "r", encoding="utf-8") as fp:
+            for line in fp:
+                if any(not x.isspace() for x in line):
+                    yield json.loads(line)
+
+
+def write_jsonl(filename: str, data: Iterable[Dict], append: bool = False):
+    """
+    Writes an iterable of dictionaries to jsonl
+    """
+    if append:
+        mode = 'ab'
+    else:
+        mode = 'wb'
+    filename = os.path.expanduser(filename)
+    if filename.endswith(".gz"):
+        with open(filename, mode) as fp:
+            with gzip.GzipFile(fileobj=fp, mode='wb') as gzfp:
+                for x in data:
+                    gzfp.write((json.dumps(x) + "\n").encode('utf-8'))
+    else:
+        with open(filename, mode) as fp:
+            for x in data:
+                fp.write((json.dumps(x) + "\n").encode('utf-8'))

DeepSeek-Coder-main/Evaluation/HumanEval/human_eval/evaluate_functional_correctness.py

@@ -0,0 +1,29 @@
+import fire
+import sys
+
+from .data import HUMAN_EVAL
+from .evaluation import evaluate_functional_correctness
+
+
+def entry_point(
+    sample_file: str,
+    k: str = "1,10,100",
+    n_workers: int = 4,
+    timeout: float = 3.0,
+    problem_file: str = "",
+    is_mbpp: bool = False,
+):
+    """
+    Evaluates the functional correctness of generated samples, and writes
+    results to f"{sample_file}_results.jsonl.gz"
+    """
+    k = list(map(int, k.split(",")))
+    results = evaluate_functional_correctness(sample_file, k, n_workers, timeout, problem_file, is_mbpp)
+    print(results)
+
+
+def main():
+    fire.Fire(entry_point)
+
+
+sys.exit(main())

DeepSeek-Coder-main/Evaluation/HumanEval/human_eval/evaluation.py

@@ -0,0 +1,296 @@
+import os
+import sys
+import fire
+import json
+import gzip
+import regex
+import numpy as np
+import itertools
+
+from typing import *
+from tqdm.auto import tqdm
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from .data import stream_jsonl
+from .execution import check_correctness
+IMPORT_HELPER = {
+    "python": [
+        "import math",
+        "import re",
+        "import sys",
+        "import copy",
+        "import datetime",
+        "import itertools",
+        "import collections",
+        "import heapq",
+        "import functools",
+        "import hashlib",
+        "import numpy",
+        "import numpy as np",
+        "import string",
+        "from typing import *",
+        "from collections import *",
+    ],
+    "go"    : [
+        "math",
+        "strings",
+        "fmt",
+        "strconv",
+        "time",
+        "bytes",
+        "regexp",
+        "sort",
+        "math/rand",
+        "crypto/md5",
+    ],
+    "cpp"   : [
+        "#include<stdlib.h>",
+        "#include<algorithm>",
+        "#include<math.h>",
+        "#include<stdio.h>",
+        "#include<vector>",
+        "#include<string>",
+        "#include<climits>",
+        "#include<cstring>",
+        "#include<iostream>",
+        "#include<cassert>"
+    ],
+    "cs": ["using System.Numerics;", "using System.Diagnostics;", "using System.Collections.Generic;", "using System.Linq;", "using System.Text;", "using System.Security.Cryptography;", "using System.Collections.Generic;"]
+}
+
+
+LANGUAGE_NAME = {
+    "cpp"   : "CPP",
+    "go"    : "Go",
+    "java"  : "Java",
+    "js"    : "JavaScript",
+    "python": "Python",
+}
+
+
+def read_dataset(
+    data_file: str = None,
+    dataset_type: str = "humaneval",
+    num_shot=None,
+) -> Dict:
+    """
+    Reads a dataset and returns a dictionary of tasks.
+    """
+    if num_shot is not None:
+        print(f"{num_shot}-shot setting...")
+    if "humaneval" in dataset_type.lower():
+        if data_file is None:
+            current_path = os.path.dirname(os.path.abspath(__file__))
+            data_file = os.path.join(current_path, "..", "humaneval-x", "python", "data", "humaneval_python.jsonl.gz")
+        dataset = {task["task_id"]: task for task in stream_jsonl(data_file)}
+    else:
+        raise f"Dataset: {dataset_type} not supported."
+
+    return dataset
+
+def estimate_pass_at_k(
+        num_samples: Union[int, List[int], np.ndarray],
+        num_correct: Union[List[int], np.ndarray],
+        k: int
+) -> np.ndarray:
+    """
+    Estimates pass@k of each problem and returns them in an array.
+    """
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """
+        Calculates 1 - comb(n - c, k) / comb(n, k).
+        """
+        if n - c < k:
+            return 1.0
+        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+    if isinstance(num_samples, int):
+        num_samples_it = itertools.repeat(num_samples, len(num_correct))
+    else:
+        assert len(num_samples) == len(num_correct)
+        num_samples_it = iter(num_samples)
+
+    return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
+
+def process_humaneval_test(sample, problems, example_test=False, is_mbpp=False, language="python"):
+    """
+    Processes a sample for evaluation.
+    """
+    task_id = sample["task_id"]
+    if is_mbpp:
+        return sample["generation"] + "\n" + "\n".join(problems[task_id]["test"])
+
+    prompt = sample["prompt"]
+    if example_test and "example_test" in problems[task_id] and problems[task_id]["example_test"] != "":
+        test = problems[task_id]["example_test"]
+    else:
+        test = problems[task_id]["test"]
+    code = sample["generation"]
+
+    # Pre-process for different languages
+    if language == "python":
+        test_setup = "\n".join(IMPORT_HELPER["python"]) + "\n"
+        test_string = test_setup + code + "\n" + test + "\n"
+    elif language == "cpp":
+        test_set_up = ""
+        for s in IMPORT_HELPER["cpp"]:
+            if s not in prompt:
+                test_set_up += s + "\n"
+        test_string = test_set_up + "\n" + code + "\n" + test
+    elif language == "java":
+        test_string = code + "\n" + test
+    elif language == "cs":
+        test_set_up = ""
+        for s in IMPORT_HELPER["cs"]:
+            test_set_up += s + "\n"
+        test_string = test_set_up + "\n" + code + "\n" + test
+    elif language in ["js", "javascript", "ts", "sh", "go"]:
+        test_string = code + "\n" + test
+    elif language == "go232":
+        import_string = problems[task_id]["import"]
+        prompt = prompt.replace(import_string, "")
+        if example_test and "example_test" in problems[task_id]:
+            test = problems[task_id]["example_test"]
+        else:
+            test = problems[task_id]["test"]
+        test_setup = problems[task_id]["test_setup"]
+        other_pkgs = []
+        for pkg in IMPORT_HELPER["go"]:
+            if pkg not in test_setup:
+                p = pkg.split("/")[-1]
+                if p + "." in code:
+                    other_pkgs.append(f"\"{pkg}\"")
+        if other_pkgs:
+            import_other_pkgs = "import (\n" + "    ".join([p + "\n" for p in other_pkgs]) + ")"
+            test_string = test_setup + "\n" + import_other_pkgs + "\n" + prompt + code + "\n" + test
+        else:
+            test_string = test_setup + "\n" + prompt + code + "\n" + test
+    elif language == "rust":
+        main = "\nfn main(){ \n } \n"
+        declaration = problems[task_id]["declaration"]
+        test_string = main + declaration + prompt + code + test
+    elif language == "php":
+        if code[:5] != "<?php":
+            code = "<?php\n" + code
+        test_string = code + "\n" + test + "?>"
+    return test_string
+
+
+def stream_jsonl_all(filename: str) -> Iterable[Dict]:
+    """
+    Streams a JSONL file.
+    """
+    results = []
+    if filename.endswith(".gz"):
+        fp = gzip.open(open(filename, "rb"), "rt")
+    else:
+        fp = open(filename, "r")
+    for line in fp:
+        if any(not x.isspace() for x in line):
+            results.append(json.loads(line))
+    fp.close()
+
+    return results
+
+
+def evaluate_functional_correctness(
+        input_file: str = None,
+        tmp_dir: str = "./",
+        n_workers: int = 32,
+        timeout: float = 10.0,
+        problem_file: str = "../data/humaneval_python.jsonl.gz",
+        out_dir: str = None,
+        k: List[int] = [1, 10, 100],
+        test_groundtruth: bool = False,
+        example_test: bool = False,
+        is_mbpp: bool = False,
+        language: str = "python",
+):
+    """
+    Evaluates the functional correctness of a model.
+    """
+    if example_test:
+        print("Example test...")
+
+    problems = read_dataset(problem_file,
+                            dataset_type="humaneval")
+    sample_jsonl = stream_jsonl_all(input_file)
+
+
+    with ThreadPoolExecutor(max_workers=n_workers) as executor:
+
+        futures = []
+        completion_id = Counter()
+        n_samples = 0
+        results = defaultdict(list)
+
+        if test_groundtruth:
+            print("Testing ground truth...")
+            for sample in tqdm(problems.values()):
+                task_id = sample["task_id"]
+                lang = task_id.split("/")[0].lower()
+                if lang == "javascript":
+                    lang = "js"
+                tmp_dir_ = os.path.join(tmp_dir, lang, "evaluation")
+                sample["generation"] = sample["canonical_solution"]
+                sample["test_code"] = process_humaneval_test(sample, problems, example_test, language)
+                if sample["test_code"] is None:
+                    continue
+                args = (task_id, sample, lang, timeout, tmp_dir_, completion_id[task_id])
+                future = executor.submit(check_correctness, *args)
+                futures.append(future)
+                completion_id[task_id] += 1
+                n_samples += 1
+        else:
+            print("Reading samples...")
+            for sample in tqdm(sample_jsonl):
+                task_id = sample["task_id"]
+                if not is_mbpp:
+                    lang = language
+                if not is_mbpp and lang == "javascript":
+                    lang = "js"
+                if is_mbpp:
+                    lang = "python"
+                tmp_dir_ = os.path.join(tmp_dir, lang, "evaluation")
+                sample["task_id"] = task_id
+                sample["test_code"] = process_humaneval_test(sample, problems, example_test, is_mbpp, language)
+                if sample["test_code"] is None:
+                    continue
+                if "completion_id" in sample:
+                    completion_id_ = sample["completion_id"]
+                else:
+                    completion_id_ = completion_id[task_id]
+                args = (task_id, sample, lang, timeout, tmp_dir_, completion_id_)
+                future = executor.submit(check_correctness, *args)
+                futures.append(future)
+                completion_id[task_id] += 1
+                n_samples += 1
+
+        if len(completion_id) == len(problems):
+            evaluate_pass_at_k = True
+        else:
+            evaluate_pass_at_k = False
+
+        print("Running test suites...")
+        for future in tqdm(as_completed(futures), total=len(futures)):
+            result = future.result()
+            results[result["task_id"]].append((result["completion_id"], result))
+
+    # Calculate pass@k.
+    total, correct = [], []
+    for result in results.values():
+        passed = [r[1]["passed"] for r in result]
+        total.append(len(passed))
+        correct.append(sum(passed))
+    total = np.array(total)
+    correct = np.array(correct)
+    if evaluate_pass_at_k:
+        ks = k
+        pass_at_k = {f"pass@{k}": estimate_pass_at_k(total, correct, k).mean()
+                     for k in ks if (total >= k).all()}
+        print(pass_at_k)
+    else:
+        print("Total:", np.sum(total))
+        print("Correct:", np.sum(correct))
+    return pass_at_k

DeepSeek-Coder-main/Evaluation/HumanEval/human_eval/execution.py

@@ -0,0 +1,731 @@
+import contextlib
+import faulthandler
+import io
+import multiprocessing
+import os
+import platform
+import signal
+import random
+import subprocess
+import tempfile
+import gzip
+import json
+from typing import *
+import traceback
+
+java_exec = ""
+node_exec = ""
+tsc_exec = ""
+go_exec = ""
+php_exec = ""
+cs_exec = ""
+
+def check_correctness(
+        task_id: str,
+        sample: dict,
+        language_type: str,
+        timeout: float = 3.0,
+        tmp_dir: str = None,
+        completion_id: Optional[int] = None,
+) -> Dict:
+    """
+    Evaluates the functional correctness of a completion by running the test
+    suite provided in the problem.
+    """
+
+    def unsafe_execute(tmp_dir):
+        random_id = random.randint(1, 100000)
+        if "python" in language_type.lower():
+            with create_tempdir():
+
+                # These system calls are needed when cleaning up tempdir.
+                import os
+                import shutil
+                rmtree = shutil.rmtree
+                rmdir = os.rmdir
+                chdir = os.chdir
+
+                # Disable functionalities that can make destructive changes to the test.
+                reliability_guard()
+
+                try:
+                    exec_globals = {}
+                    with swallow_io():
+                        with time_limit(timeout):
+                            # WARNING
+                            # This program exists to execute untrusted model-generated code. Although
+                            # it is highly unlikely that model-generated code will do something overtly
+                            # malicious in response to this test suite, model-generated code may act
+                            # destructively due to a lack of model capability or alignment.
+                            # Users are strongly encouraged to sandbox this evaluation suite so that it
+                            # does not perform destructive actions on their host or network.
+                            # Once you have read this disclaimer and taken appropriate precautions,
+                            # uncomment the following line and proceed at your own risk:
+                            exec(sample["test_code"], exec_globals)
+                        result.append("passed")
+                except TimeoutException:
+                    result.append("timed out")
+                except AssertionError as e:
+                    result.append(f"failed: AssertionError")
+                except BaseException as e:
+                    result.append(f"failed: {e}")
+                #print(sample["test_code"])
+                #print(result)
+                # Needed for cleaning up.
+                shutil.rmtree = rmtree
+                os.rmdir = rmdir
+                os.chdir = chdir
+
+        elif "go" in language_type.lower():
+            assert tmp_dir is not None, "Go should be evaluated in a dir where necessary module files installed."
+
+            import os
+            import shutil
+
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            origin_path = os.getcwd()
+            os.chdir(tmp_dir)
+            open(f"main_test.go", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                     exec_result = subprocess.run([f"{go_exec}go", "test", f"-timeout={timeout}s", "main_test.go"], timeout=timeout, capture_output=True)
+
+                if exec_result.returncode == 0:
+                    result.append("passed")
+                else:
+                    if exec_result.stderr:
+                        try:
+                            err = exec_result.stderr.decode()
+                        except:
+                            err = exec_result.stderr
+                    else:
+                        try:
+                            err = exec_result.stdout.decode()
+                        except:
+                            err = exec_result.stdout
+                    result.append(f"failed: {err}")
+
+            except TimeoutException:
+                result.append("timed out")
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "js" in language_type.lower():
+            import os
+            import shutil
+
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            origin_path = os.getcwd()
+            os.chdir(tmp_dir)
+            open(f"test.js", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                     exec_result = subprocess.run([f"{node_exec}node", "test.js"], timeout=timeout, capture_output=True)
+
+                if exec_result.stderr.decode():
+                    err = exec_result.stderr.decode()
+                    result.append(f"failed: {err}")
+                elif exec_result.stdout.decode():
+                    err = exec_result.stdout.decode()
+                    result.append(f"failed: {err}")
+                else:
+                    result.append("passed")
+
+            except TimeoutException:
+                result.append("timed out")
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "cpp" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            open(f"test.cpp", 'w').write(sample["test_code"])
+            if "162" in task_id:
+                compilation_result = subprocess.run(["/usr/bin/g++", "-std=c++17", "test.cpp", "-lcrypto", "-lssl"],
+                                                    timeout=timeout,
+                                                    capture_output=True)
+            else:
+                compilation_result = subprocess.run(["/usr/bin/g++", "-std=c++17", "test.cpp"], timeout=timeout,
+                                                    capture_output=True)
+            if compilation_result.returncode != 0:
+                if compilation_result.stderr:
+                    err = compilation_result.stderr.decode()
+                else:
+                    err = compilation_result.stdout.decode()
+                result.append(f"failed: compilation error: {err}")
+            else:
+                try:
+                    exec_result = None
+                    with time_limit(timeout):
+                        # WARNING
+                        # This program exists to execute untrusted model-generated code. Although
+                        # it is highly unlikely that model-generated code will do something overtly
+                        # malicious in response to this test suite, model-generated code may act
+                        # destructively due to a lack of model capability or alignment.
+                        # Users are strongly encouraged to sandbox this evaluation suite so that it
+                        # does not perform destructive actions on their host or network.
+                        # Once you have read this disclaimer and taken appropriate precautions,
+                        # uncomment the following line and proceed at your own risk:
+                         exec_result = subprocess.run(["./a.out"], timeout=timeout, capture_output=True)
+
+                    if exec_result.returncode == 0:
+                        result.append("passed")
+                    else:
+                        if exec_result.stderr:
+                            try:
+                                err = exec_result.stderr.decode()
+                            except:
+                                err = exec_result.stderr
+                        else:
+                            try:
+                                err = exec_result.stdout.decode()
+                            except:
+                                err = exec_result.stdout
+                        result.append(f"failed: {err}")
+                except TimeoutException:
+                    result.append("timed out")
+            #print(result[-1])
+            #print(sample["test_code"])
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "php" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            open(f"test.php", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    cmd = f"{php_exec}php -f test.php"
+                    exec_result = subprocess.run(cmd, timeout=timeout, capture_output=True, shell=True)
+
+                if exec_result.returncode == 0:
+                    result.append("passed")
+                else:
+                    if exec_result.stderr:
+                        try:
+                            err = exec_result.stderr.decode()
+                        except:
+                            err = exec_result.stderr
+                    else:
+                        try:
+                            err = exec_result.stdout.decode()
+                        except:
+                            err = exec_result.stdout
+                    result.append(f"failed: {err}")
+            except TimeoutException:
+                result.append("timed out")
+            print(result[-1])
+            print(sample["test_code"])
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "sh" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            open(f"test.sh", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    cmd = "/bin/bash test.sh"
+                    exec_result = subprocess.run(cmd, timeout=10, capture_output=True, shell=True)
+
+                if exec_result.returncode == 0:
+                    result.append("passed")
+                else:
+                    if exec_result.stderr:
+                        try:
+                            err = exec_result.stderr.decode()
+                        except:
+                            err = exec_result.stderr
+                    else:
+                        try:
+                            err = exec_result.stdout.decode()
+                        except:
+                            err = exec_result.stdout
+                    result.append(f"failed: {err}")
+            except TimeoutException:
+                result.append("timed out")
+            #print(result[-1])
+            #print(sample["test_code"])
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "ts" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            env = {"PATH": f"{node_exec}:" + subprocess.os.environ["PATH"]}
+            open(f"test.ts", 'w').write(sample["test_code"])
+            cmd = f"{tsc_exec}tsc test.ts --target ES2015 --lib ES2015,DOM"
+            compilation_result = subprocess.run(cmd, timeout=timeout, capture_output=True, env=env, shell=True)
+            if compilation_result.returncode != 0:
+                if compilation_result.stderr:
+                    err = compilation_result.stderr.decode()
+                else:
+                    err = compilation_result.stdout.decode()
+                result.append(f"failed: compilation error: {err}")
+            else:
+                try:
+                    exec_result = None
+                    with time_limit(timeout):
+                         exec_result = subprocess.run([f"{node_exec}node", "test.js"], timeout=timeout, capture_output=True)
+
+                    if exec_result.returncode == 0:
+                        result.append("passed")
+                    else:
+                        if exec_result.stderr:
+                            try:
+                                err = exec_result.stderr.decode()
+                            except:
+                                err = exec_result.stderr
+                        else:
+                            try:
+                                err = exec_result.stdout.decode()
+                            except:
+                                err = exec_result.stdout
+                        result.append(f"failed: {err}")
+                except TimeoutException:
+                    result.append("timed out")
+            if result[-1] != "passed":
+                env = {"PATH": f"{node_exec}:" + subprocess.os.environ["PATH"]}
+                cmd = f"{tsc_exec}tsc test.ts"
+                compilation_result = subprocess.run(cmd, timeout=timeout, capture_output=True, env=env, shell=True)
+                if compilation_result.returncode != 0:
+                    if compilation_result.stderr:
+                        err = compilation_result.stderr.decode()
+                    else:
+                        err = compilation_result.stdout.decode()
+                    result[-1] = f"failed: compilation error: {err}"
+                else:
+                    try:
+                        exec_result = None
+                        with time_limit(timeout):
+                            exec_result = subprocess.run([f"{node_exec}node", "test.js"], timeout=timeout, capture_output=True)
+
+                        if exec_result.returncode == 0:
+                            result[-1] = "passed"
+                        else:
+                            if exec_result.stderr:
+                                try:
+                                    err = exec_result.stderr.decode()
+                                except:
+                                    err = exec_result.stderr
+                            else:
+                                try:
+                                    err = exec_result.stdout.decode()
+                                except:
+                                    err = exec_result.stdout
+                            result[-1] = f"failed: {err}"
+                    except TimeoutException:
+                        result[-1] = "timed out"
+ 
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "cs" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            os.chdir(tmp_dir)
+            open(f"Program.cs", 'w').write(sample["test_code"])
+            cmd = f"{cs_exec}mcs -d:DEBUG Program.cs"
+            compilation_result = subprocess.run(cmd, shell=True, capture_output=True)
+            if compilation_result.returncode != 0:
+                if compilation_result.stderr:
+                    err = compilation_result.stderr.decode()
+                else:
+                    err = compilation_result.stdout.decode()
+                result.append(f"failed: compilation error: {err}")
+            else:
+                try:
+                    exec_result = None
+                    cmd = f"{cs_exec}mono Program.exe"
+                    env = dict(MONO_TRACE_LISTENER="Console.Error")
+                    with time_limit(timeout):
+                         exec_result = subprocess.run(cmd, timeout=timeout, shell=True, capture_output=True, env=env)
+
+                    if "Fail" not in exec_result.stderr.decode():
+                        result.append("passed")
+                    else:
+                        if exec_result.stderr:
+                            try:
+                                err = exec_result.stderr.decode()
+                            except:
+                                err = exec_result.stderr
+                        else:
+                            try:
+                                err = exec_result.stdout.decode()
+                            except:
+                                err = exec_result.stdout
+                        result.append(f"failed: {err}")
+                except TimeoutException:
+                    result.append("timed out")
+                except Exception as e:
+                    result.append(f"failed: {e}")
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "rust" in language_type.lower():  
+            import os         
+            
+            WD: str = os.path.dirname(os.path.abspath(__file__))
+            RUST_DIR: str = os.path.join(WD, "rust")
+            RUST_SRC: str = os.path.join(RUST_DIR, "src")
+            RUST_BIN: str = os.path.join(RUST_SRC, "bin")
+            RUST_TMP_DIR: str = os.path.join(RUST_DIR, "tmp")
+            RUST_LOGS: str = os.path.join(RUST_TMP_DIR, "logs")
+            RUST_EXT: str = ".rs" 
+
+            # Create mandatory tmp directories
+            os.makedirs(RUST_TMP_DIR, exist_ok=True)
+            os.makedirs(RUST_LOGS, exist_ok=True)
+            os.makedirs(RUST_SRC, exist_ok=True)
+            os.makedirs(RUST_BIN, exist_ok=True)
+
+            with tempfile.NamedTemporaryFile(dir = RUST_BIN, delete=False) as f:
+                #temporal file name
+                file_prefix = sample["task_id"].lower().replace("/", "_")
+                file_name:str =  file_prefix +RUST_EXT
+                
+                os.rename(f.name, os.path.join(RUST_BIN, file_name))
+                
+                # Sample to pure Rust function
+                rust_code: str = sample["test_code"]
+
+                # dump the rust source code in the target temporal file
+                f.write(rust_code.encode('utf-8'))
+
+            # Proceed towards Rust binaries compilation. Therefore move to Rust module root dir.
+            os.chdir(RUST_DIR)
+
+            # Two possible outcomes
+            # Pass OR Fail compilation
+            log_filename: str = file_prefix + ".jsonl"
+            log_path: str = os.path.join(RUST_LOGS, log_filename)
+            cargo_check: str = "cargo check --bin " + file_prefix + " --message-format json >> " + log_path
+            # Compilation build status
+            returned_val_compilation: int
+            
+            # Overwrite file content
+            if os.path.exists(log_path):
+                if(file_size := os.path.getsize(log_path)) >= 0: 
+                    os.remove(log_path)
+                    returned_val_compilation = os.system(cargo_check)
+
+            else: 
+                returned_val_compilation = os.system(cargo_check)
+
+            # 0 means success   
+            if returned_val_compilation == 0:
+
+                #Execution pipeline
+                cargo_test: str = "cargo test --bin " +file_prefix+ " --message-format json >> " + log_path
+                returned_val_execution = os.system(cargo_test)
+                
+                if returned_val_execution == 0:
+                    result.append("passed")
+                else:
+                   result.append(f"failed: execution error") 
+
+            else:
+                result.append(f"failed: compilation error")
+
+
+        elif "java" in language_type.lower():
+            assert tmp_dir is not None, "Java should be evaluated in a temporary dir."
+
+            import os
+            import shutil
+
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            open(os.path.join(tmp_dir, "Problem.java"), 'w').write(sample["test_code"])
+            origin_path = os.getcwd()
+            os.system(f"cp ./javatuples-1.2.jar {tmp_dir}/")
+            os.chdir(tmp_dir)
+            res = "failed: unknown error"
+            compile_returncode = -1
+            for _ in range(5):
+                try:
+                    cmd = f"{java_exec}javac -cp javatuples-1.2.jar Problem.java"
+                    compilation_result = subprocess.run(cmd, timeout=60, capture_output=True, shell=True)  
+                    compile_returncode = compilation_result.returncode
+                    break
+                except subprocess.TimeoutExpired as e:
+                    continue
+            if compile_returncode != 0:
+                res = "failed: compilation error"
+            else:
+                exec_result = None
+                try:
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                    cmd = f"{java_exec}java -ea -cp .:javatuples-1.2.jar Problem"
+                    exec_result = subprocess.run(cmd, timeout=timeout, capture_output=True, shell=True)  
+                    if exec_result.returncode == 0:
+                        res = "passed"
+                    elif exec_result.returncode == 1:
+                        if "AssertionError" in exec_result.stderr.decode('unicode-escape'):
+                            res = "failed: wrong answer"
+                        else:
+                            res = f"failed: {exec_result.stderr.decode()}"
+                except subprocess.TimeoutExpired as e:
+                    res = "time out"
+                except BaseException as e:
+                    res = f"failed: {e}"
+
+            result.append(res)  
+            os.chdir(origin_path)          
+            shutil.rmtree(tmp_dir)
+        
+    manager = multiprocessing.Manager()
+    result = manager.list()
+
+    p = multiprocessing.Process(target=unsafe_execute, args=(tmp_dir,))
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+
+    if not result:
+        result.append("timed out")
+
+    return {
+        "task_id"      : task_id,
+        "completion_id": completion_id,
+        "result"       : result[0],
+        "passed"       : result[0] == "passed",
+        "finish"       : -1 if "finish" not in sample else sample["finish"],
+        "code"         : sample["test_code"]
+    }
+
+# Copyright (c) OpenAI (https://openai.com)
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# ============================================================================
+@contextlib.contextmanager
+def time_limit(seconds: float):
+    def signal_handler(signum, frame):
+        raise TimeoutException("Timed out!")
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def swallow_io():
+    stream = WriteOnlyStringIO()
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """ StringIO that throws an exception when it's read from """
+
+    def read(self, *args, **kwargs):
+        raise IOError
+
+    def readline(self, *args, **kwargs):
+        raise IOError
+
+    def readlines(self, *args, **kwargs):
+        raise IOError
+
+    def readable(self, *args, **kwargs):
+        """ Returns True if the IO object can be read. """
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = 'stdin'
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == ".":
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes: Optional[int] = None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the 
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == 'Darwin':
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+    os.environ['OMP_NUM_THREADS'] = '1'
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__['help'] = None
+
+    import sys
+    sys.modules['ipdb'] = None
+    sys.modules['joblib'] = None
+    sys.modules['resource'] = None
+    sys.modules['psutil'] = None
+    sys.modules['tkinter'] = None

DeepSeek-Coder-main/Evaluation/HumanEval/humaneval.py

@@ -0,0 +1,163 @@
+import time
+import string
+import multiprocessing
+import os
+import numpy as np
+import json
+import re
+import torch
+import datetime
+import subprocess
+import torch.distributed as dist
+from attrdict import AttrDict
+from human_eval.evaluation import evaluate_functional_correctness
+from transformers import AutoTokenizer
+from utils.dataset import HumanEvalDataset
+from utils.utils import cleanup_code
+
+class HumanEval:
+    """
+    HumanEval evaluation class.
+    """
+    def __init__(self, data_root, max_seq_len=2048,
+                language="python", max_gen_len=200, batch_size=512,
+                log_dir=None, temperature=0, issft=False, top_p=0.95,
+                model_name="", inference_increment=True,
+                tokenizer_cfg=None, n_sample=40, k_sample=1):
+        self.data_root = data_root
+        self.max_seq_len = max_seq_len
+        self.max_gen_len = max_gen_len
+        self.batch_size = batch_size
+        self.k = k_sample
+        self.n_sample = n_sample
+        self.language = language
+        self.log_dir = log_dir
+        self.sft = issft
+        self.temperature = temperature
+        self.top_p = top_p
+        self.model_name = tokenizer_cfg["model_path"].replace("/", "_")
+        self.inference_increment = inference_increment
+        os.makedirs(self.log_dir, exist_ok=True)
+        tokenizer_cls = tokenizer_cfg.pop('cls')
+        try:
+            self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_cfg.pop("model_path"), trust_remote_code=True)       
+        except Exception as e:
+            print(e)
+            assert False
+
+    @torch.no_grad()
+    def eval_model(self, gpt, accelerator):
+        """
+        Evaluate the model on HumanEval.
+        """
+        assert self.log_dir is not None, "log_dir should not be None when evaluating humaneval"
+        dataset = HumanEvalDataset(self.data_root, sample_num=self.n_sample, language=self.language, issft=self.sft)
+        nprompt = len(dataset) // self.n_sample
+        dp_rank = accelerator.process_index 
+        dp_size = accelerator.num_processes 
+        if self.k > 1:
+            assert self.n_sample >= 100, "HumanEval PASS@100 needs n_sample >= 100"
+        gpt.eval()
+        # each process will process a subset of the dataset
+        prompt_indices_split = np.array_split(range(nprompt), dp_size)
+        prompt_indices = prompt_indices_split[dp_rank]
+        indices = [x * self.n_sample + j for x in prompt_indices for j in range(self.n_sample)]
+        all_num = len(indices) 
+        processed_num = 0
+        log_file = os.path.join(self.log_dir,
+                                    f'{self.model_name}_rank{dp_rank}_bs{self.batch_size}_shot_log_{self.language}.json')
+        tmpfile = open(log_file, "w")
+        start_time = time.time()
+        # split the dataset into batches and construct a list of inputs
+        for idx in range(0, len(indices), self.batch_size):
+            prompt_list = []
+            prompt_lens = []
+            orriginal_prompt_list = []
+            tokenized_prompt_lens = []
+            taskid = []
+            # get the prompts from the dataset
+            for j in indices[idx:idx + self.batch_size]:
+                data = dataset[j]
+                fprompt = data["prompt"].strip()
+                prompt_list.append(fprompt)
+                tmp = self.tokenizer.encode(fprompt)
+                orriginal_prompt_list.append(data["original_prompt"])
+                prompt_lens.append(len(fprompt))
+                tokenized_prompt_lens.append(tmp)
+                taskid.append(data["task_id"])
+            input_ids = torch.tensor(tokenized_prompt_lens).to(accelerator.device)
+            # generate the code
+            if self.temperature != 0:       
+                decoded = gpt.generate(
+                    input_ids=input_ids,
+                    max_new_tokens=self.max_gen_len,
+                    do_sample=True,
+                    eos_token_id=self.tokenizer.eos_token_id,
+                    temperature=self.temperature,
+                    top_p=self.top_p,
+                    pad_token_id=self.tokenizer.eos_token_id,
+                )
+            else:
+                decoded = gpt.generate(
+                    input_ids=input_ids,
+                    max_new_tokens=self.max_gen_len,
+                    do_sample=False,
+                    eos_token_id=self.tokenizer.eos_token_id,
+                    pad_token_id=self.tokenizer.eos_token_id,
+                )
+            # save the results to a file
+            for local_idx, text in enumerate(decoded):
+                prediction = decoded[local_idx]
+                prediction = self.tokenizer.decode(prediction, skip_special_tokens=True)
+                suffixprediction = prediction[prompt_lens[local_idx]:]
+                suffixprediction = cleanup_code(suffixprediction, self.language, "humaneval", self.sft, dataset.stopwords)
+                # sft mode does not need original prompt
+                if not self.sft:
+                    suffixprediction = orriginal_prompt_list[local_idx] + "\n" + suffixprediction
+                res = {"task_id": taskid[local_idx], "generation": suffixprediction, "prompt": orriginal_prompt_list[local_idx], "wholecode":prediction}
+                tmpfile.write(json.dumps(res) + "\n")
+                tmpfile.flush()
+                processed_num += 1
+            self.log_score(dp_rank, processed_num, all_num, start_time, self.batch_size)
+        tmpfile.close()        
+        accelerator.wait_for_everyone()
+        # calculate the final score of pass@k
+        self._calculate_final_score(accelerator)
+        accelerator.wait_for_everyone()
+        return
+    
+    def log_score(self, dp_rank, processed_num, all_num, start_time, bs):
+        """
+        Log the score.
+        """
+        mem = torch.cuda.max_memory_allocated() / (1 << 30)
+        avg_time = (time.time() - start_time) / processed_num * bs
+        print(
+            f'DP RANK:{dp_rank} process_num/all_num:{int(processed_num)}/{all_num} '
+            f'avg_time_per_batch:{avg_time:.2f} s '
+            f'still_need:{((all_num - processed_num) // bs + 1) * avg_time / 60:.2f} m',
+            f'mem:{mem:.3f} GiB bs:{bs}',
+            flush=True
+        )
+        if processed_num == all_num:
+            print(f'EVAL DONE! Process time {(time.time() - start_time) / 60:.2f} m', flush=True)
+    
+    def _calculate_final_score(self, accelerator):
+        """
+        Calculate the final score.
+        """
+        if accelerator.is_local_main_process:
+            logfilepath = os.path.join(self.log_dir, f'final_{self.model_name}.jsonl')
+            logfile = open(logfilepath, "w")
+            for i in range(accelerator.num_processes):
+                tmplogfile = os.path.join(self.log_dir, f'{self.model_name}_rank{i}_bs{self.batch_size}_shot_log_{self.language}.json')
+                logfile.write(open(tmplogfile).read().strip() + "\n")
+                os.remove(tmplogfile)
+            logfile.close()
+            timeout = 10
+            runlang = self.language
+            res = evaluate_functional_correctness(input_file=logfilepath, problem_file=os.path.join(self.data_root, f"humaneval-{self.language}.jsonl"), tmp_dir=self.log_dir, timeout=timeout, language=runlang)
+            print("score is", res['pass@%d' % self.k])
+            os.remove(logfilepath)
+        return
+            

DeepSeek-Coder-main/Evaluation/HumanEval/test_config.yaml

@@ -0,0 +1,15 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: MULTI_GPU
+downcast_bf16: 'no'
+gpu_ids: all
+machine_rank: 0
+main_training_function: main
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 3
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

DeepSeek-Coder-main/Evaluation/HumanEval/utils/dataset.py

@@ -0,0 +1,61 @@
+import os
+import numpy as np
+import json
+
+class HumanEvalDataset:
+
+    def __init__(self, root, sample_num=1, language="python", issft=False):
+        """
+        root: the path to the HumanEval dataset
+        sample_num: the number of samples for each prompt
+        language: the language of the HumanEval dataset
+        issft: whether to use the SFT setting
+        """
+        self.root = root
+        self.data = open(os.path.join(self.root, f"humaneval-{language}.jsonl")).readlines()
+
+        tmp = self.get_qa_only_data(self.data, issft)
+        self.clean_data = []
+        for i in range(len(tmp)):
+            for j in range(sample_num):
+                self.clean_data.append(tmp[i])
+        self.stopwords = self.clean_data[0]["stopwords"]
+        np.random.seed(1234)
+        print(f"Read HumanEval from {root}, number of samples {len(self.clean_data)}")
+
+    def get_qa_only_data(self, data_json, sft=False):
+        """
+        data_json: the jsonl file of HumanEval
+        sft: whether to use the SFT setting
+        return: a list of dict, each dict contains the prompt, task_id and stopwords
+        """
+        ans = []
+        for line in data_json:
+            line = json.loads(line)
+            prompt = line["prompt"].strip()
+            if "prefix" in line:
+                origin_prompt = line["prefix"]
+            else:
+                origin_prompt = line["prompt"]
+
+            if sft:
+                prompt = f"""Below is an instruction that describes a task, paired with an input that provides further context.\nWrite a response that appropriately completes the request.\n\n### Instruction:\nWrite a program to perform the given task.\n\nInput:\n{prompt}\n\n### Response:\n"""
+            if "stop_tokens" in line:
+                s = line["stop_tokens"]
+            else:
+                s = []
+            ans.append({"prompt":prompt, "task_id":line["task_id"], "original_prompt": origin_prompt, "stopwords":s})
+        return ans
+
+    def __len__(self):
+        """
+        return the number of samples in the dataset
+        """
+        return len(self.clean_data)
+
+    def __getitem__(self, index):
+        """
+        return the sample at index
+        """
+        sample = self.clean_data[index]
+        return sample

DeepSeek-Coder-main/Evaluation/HumanEval/utils/utils.py

@@ -0,0 +1,146 @@
+import re
+
+languge_settings = {
+    'python': {
+        'full_name': 'Python',
+        'indent': 4,
+    },
+    'cpp': {
+        'full_name': 'cpp',
+        'indent': 0,
+        'main': "int main()",
+    },
+    'java': {
+        'full_name': 'Java',
+        'indent': 4,
+        'main': "public static void main",
+    },
+    'cs': {
+        'full_name': "csharp",
+        'indent': 0,
+        'main': "public static void Main",
+    },
+    'php': {
+        'full_name': "PHP",
+        'indent': 0,
+    },
+    'ts': {
+        'full_name': "TypeScript",
+        'indent': 0,
+    },
+    'js': {
+        'full_name': "JavaScript",
+        'indent': 0
+    },
+    'sh': {
+        'full_name': "Bash",
+        'indent': 0
+    }
+}
+
+def get_function_name(question: str, lang: str):
+    func_lines = [x for x in question.strip().split('\n') if x.strip()]
+
+    if lang.lower() == 'python':
+        func_idx = [i for i in range(len(func_lines)) if func_lines[i].startswith("def ")][-1]
+        func_name = func_lines[func_idx].split('(')[0].strip()
+        func_prefix = "\n".join(func_lines[:func_idx])
+        return func_name, func_prefix
+    
+    func_name = func_lines[-1].split('{')[0].strip()
+    func_prefix = "\n".join(func_lines[:-1])
+    return func_name, func_prefix
+
+def extract_generation_code(example: str, lang_code: str, verbose: bool=False):
+    task_id = example['task_id']
+    output = example.get('output', example.get("gpt_completion"))
+    question = example["prompt"].strip()
+    setting = languge_settings[lang_code]
+    lang = setting['full_name']
+    indent = setting['indent']
+
+    try:
+        code_block: str = re.findall(f'```{lang.lower()}\n(.*?)```', output, re.DOTALL | re.IGNORECASE)[0]
+        if verbose:
+            print(">>> Task: {}\n{}".format(task_id, code_block))
+        
+        # Remove main
+        if setting.get('main', None) and setting['main'] in code_block:
+            main_start = code_block.index(setting['main'])
+            code_block = code_block[:main_start]
+        
+        func_name, func_prefix = get_function_name(question, lang)
+
+        try:
+            start = code_block.lower().index(func_name.lower())
+            indent = 0
+            while start - indent >= 0 and code_block[start - indent-1] == ' ':
+                indent += 1
+            
+            try:
+                end = code_block.rindex('\n' + ' '*indent + '}')
+            except:
+                end = len(code_block)
+        except:
+            start = 0
+            try:
+                end = code_block.rindex('\n' + ' '*indent + '}')
+            except:
+                end = len(code_block)
+
+        body = code_block[start:end]
+
+        if lang_code.lower() in ['php', 'ts', 'js']:
+            body += '\n' + ' '*indent + '}'
+    
+        generation = func_prefix + '\n' + body + '\n'
+        example['generation'] = generation
+
+    except Exception as ex:
+        print("Failed to extract code block with error `{}`:\n>>> Task: {}\n>>> Output:\n{}".format(
+            ex, task_id, output
+        ))
+        example['generation'] = example['prompt'] + '\n' + output
+    
+    return example
+
+def cleanup_code(
+    code: str,
+    language_type: str = None,
+    dataset: str = None,
+    issft: bool = False,
+    stop_words = []
+):
+    """
+    Cleans up the generated code.
+    """
+
+    if language_type.lower() == "python":
+        if issft:
+            code = _clean_python_code_for_sft(code)
+        stop_words = ["\ndef", "\nclass", "\nif", "\n#", "\nprint"]
+        code = _truncate_code_at_stopwords(code, stop_words)
+    elif language_type.lower() == "ts":
+        code = _truncate_code_at_stopwords(code, stop_words + ["\nexport", "\nimport", "\nexport default", "\nimport default", "\nconsole.log"])
+    else:
+        code = _truncate_code_at_stopwords(code, stop_words)
+
+    return code
+
+def _clean_python_code_for_sft(code):
+    code = code.replace("\r", "")
+    if "```python" in code:
+        code_start_idx = code.index("```python")
+        code = code[code_start_idx:].replace("```python", "").strip()
+        end_idx = code.find("```") if "```" in code else len(code)
+        code = code[:end_idx].strip()
+
+    return code
+
+def _truncate_code_at_stopwords(code, stop_words):
+    min_stop_idx = len(code)
+    for stop_word in stop_words:
+        stop_index = code.find(stop_word)
+        if 0 <= stop_index < min_stop_idx:
+            min_stop_idx = stop_index
+    return code[:min_stop_idx]