From 0343fa953cbad5f02f105d9e5a3ca4480c1647cc Mon Sep 17 00:00:00 2001 From: imamnurby Date: Thu, 11 Jul 2024 12:32:47 +0800 Subject: [PATCH 01/14] add ids that do not use the network functionality --- bigcodebench/data/network-free-set.txt | 517 +++++++++++++++++++++++++ 1 file changed, 517 insertions(+) create mode 100644 bigcodebench/data/network-free-set.txt diff --git a/bigcodebench/data/network-free-set.txt b/bigcodebench/data/network-free-set.txt new file mode 100644 index 0000000..2d43724 --- /dev/null +++ b/bigcodebench/data/network-free-set.txt @@ -0,0 +1,517 @@ +BigCodeBench/0 +BigCodeBench/1 +BigCodeBench/2 +BigCodeBench/3 +BigCodeBench/4 +BigCodeBench/5 +BigCodeBench/6 +BigCodeBench/7 +BigCodeBench/8 +BigCodeBench/9 +BigCodeBench/10 +BigCodeBench/11 +BigCodeBench/12 +BigCodeBench/14 +BigCodeBench/15 +BigCodeBench/16 +BigCodeBench/17 +BigCodeBench/18 +BigCodeBench/19 +BigCodeBench/20 +BigCodeBench/21 +BigCodeBench/22 +BigCodeBench/23 +BigCodeBench/24 +BigCodeBench/25 +BigCodeBench/26 +BigCodeBench/27 +BigCodeBench/29 +BigCodeBench/30 +BigCodeBench/31 +BigCodeBench/33 +BigCodeBench/34 +BigCodeBench/35 +BigCodeBench/36 +BigCodeBench/37 +BigCodeBench/38 +BigCodeBench/39 +BigCodeBench/40 +BigCodeBench/41 +BigCodeBench/42 +BigCodeBench/43 +BigCodeBench/44 +BigCodeBench/45 +BigCodeBench/46 +BigCodeBench/47 +BigCodeBench/48 +BigCodeBench/49 +BigCodeBench/50 +BigCodeBench/51 +BigCodeBench/52 +BigCodeBench/53 +BigCodeBench/54 +BigCodeBench/55 +BigCodeBench/56 +BigCodeBench/57 +BigCodeBench/58 +BigCodeBench/60 +BigCodeBench/61 +BigCodeBench/62 +BigCodeBench/63 +BigCodeBench/64 +BigCodeBench/65 +BigCodeBench/66 +BigCodeBench/67 +BigCodeBench/68 +BigCodeBench/69 +BigCodeBench/70 +BigCodeBench/71 +BigCodeBench/72 +BigCodeBench/73 +BigCodeBench/75 +BigCodeBench/84 +BigCodeBench/85 +BigCodeBench/86 +BigCodeBench/87 +BigCodeBench/88 +BigCodeBench/89 +BigCodeBench/90 +BigCodeBench/91 +BigCodeBench/92 +BigCodeBench/93 +BigCodeBench/94 +BigCodeBench/95 +BigCodeBench/96 +BigCodeBench/97 +BigCodeBench/98 +BigCodeBench/99 +BigCodeBench/100 +BigCodeBench/102 +BigCodeBench/103 +BigCodeBench/104 +BigCodeBench/105 +BigCodeBench/106 +BigCodeBench/107 +BigCodeBench/108 +BigCodeBench/109 +BigCodeBench/110 +BigCodeBench/111 +BigCodeBench/112 +BigCodeBench/113 +BigCodeBench/114 +BigCodeBench/115 +BigCodeBench/116 +BigCodeBench/117 +BigCodeBench/118 +BigCodeBench/119 +BigCodeBench/120 +BigCodeBench/121 +BigCodeBench/122 +BigCodeBench/123 +BigCodeBench/124 +BigCodeBench/125 +BigCodeBench/126 +BigCodeBench/127 +BigCodeBench/128 +BigCodeBench/130 +BigCodeBench/131 +BigCodeBench/132 +BigCodeBench/133 +BigCodeBench/134 +BigCodeBench/135 +BigCodeBench/136 +BigCodeBench/137 +BigCodeBench/138 +BigCodeBench/139 +BigCodeBench/140 +BigCodeBench/141 +BigCodeBench/142 +BigCodeBench/143 +BigCodeBench/148 +BigCodeBench/149 +BigCodeBench/150 +BigCodeBench/151 +BigCodeBench/152 +BigCodeBench/153 +BigCodeBench/155 +BigCodeBench/156 +BigCodeBench/157 +BigCodeBench/159 +BigCodeBench/160 +BigCodeBench/161 +BigCodeBench/162 +BigCodeBench/163 +BigCodeBench/164 +BigCodeBench/165 +BigCodeBench/166 +BigCodeBench/167 +BigCodeBench/168 +BigCodeBench/169 +BigCodeBench/171 +BigCodeBench/172 +BigCodeBench/173 +BigCodeBench/174 +BigCodeBench/175 +BigCodeBench/177 +BigCodeBench/178 +BigCodeBench/179 +BigCodeBench/180 +BigCodeBench/182 +BigCodeBench/184 +BigCodeBench/187 +BigCodeBench/190 +BigCodeBench/191 +BigCodeBench/193 +BigCodeBench/194 +BigCodeBench/196 +BigCodeBench/197 +BigCodeBench/198 +BigCodeBench/199 +BigCodeBench/200 +BigCodeBench/201 +BigCodeBench/202 +BigCodeBench/204 +BigCodeBench/205 +BigCodeBench/206 +BigCodeBench/208 +BigCodeBench/209 +BigCodeBench/210 +BigCodeBench/212 +BigCodeBench/213 +BigCodeBench/214 +BigCodeBench/216 +BigCodeBench/217 +BigCodeBench/218 +BigCodeBench/219 +BigCodeBench/220 +BigCodeBench/221 +BigCodeBench/222 +BigCodeBench/223 +BigCodeBench/224 +BigCodeBench/225 +BigCodeBench/226 +BigCodeBench/227 +BigCodeBench/228 +BigCodeBench/229 +BigCodeBench/230 +BigCodeBench/231 +BigCodeBench/232 +BigCodeBench/233 +BigCodeBench/234 +BigCodeBench/235 +BigCodeBench/236 +BigCodeBench/237 +BigCodeBench/238 +BigCodeBench/239 +BigCodeBench/240 +BigCodeBench/241 +BigCodeBench/242 +BigCodeBench/243 +BigCodeBench/244 +BigCodeBench/245 +BigCodeBench/246 +BigCodeBench/247 +BigCodeBench/248 +BigCodeBench/249 +BigCodeBench/250 +BigCodeBench/251 +BigCodeBench/252 +BigCodeBench/253 +BigCodeBench/254 +BigCodeBench/255 +BigCodeBench/256 +BigCodeBench/257 +BigCodeBench/258 +BigCodeBench/259 +BigCodeBench/260 +BigCodeBench/261 +BigCodeBench/262 +BigCodeBench/263 +BigCodeBench/264 +BigCodeBench/265 +BigCodeBench/266 +BigCodeBench/267 +BigCodeBench/268 +BigCodeBench/269 +BigCodeBench/270 +BigCodeBench/271 +BigCodeBench/275 +BigCodeBench/276 +BigCodeBench/277 +BigCodeBench/278 +BigCodeBench/279 +BigCodeBench/280 +BigCodeBench/281 +BigCodeBench/282 +BigCodeBench/283 +BigCodeBench/286 +BigCodeBench/287 +BigCodeBench/288 +BigCodeBench/289 +BigCodeBench/291 +BigCodeBench/292 +BigCodeBench/293 +BigCodeBench/294 +BigCodeBench/295 +BigCodeBench/296 +BigCodeBench/297 +BigCodeBench/298 +BigCodeBench/299 +BigCodeBench/300 +BigCodeBench/301 +BigCodeBench/302 +BigCodeBench/303 +BigCodeBench/304 +BigCodeBench/305 +BigCodeBench/306 +BigCodeBench/307 +BigCodeBench/308 +BigCodeBench/309 +BigCodeBench/310 +BigCodeBench/311 +BigCodeBench/312 +BigCodeBench/313 +BigCodeBench/316 +BigCodeBench/317 +BigCodeBench/318 +BigCodeBench/319 +BigCodeBench/320 +BigCodeBench/321 +BigCodeBench/322 +BigCodeBench/323 +BigCodeBench/324 +BigCodeBench/325 +BigCodeBench/326 +BigCodeBench/327 +BigCodeBench/328 +BigCodeBench/329 +BigCodeBench/330 +BigCodeBench/331 +BigCodeBench/333 +BigCodeBench/334 +BigCodeBench/335 +BigCodeBench/336 +BigCodeBench/337 +BigCodeBench/338 +BigCodeBench/340 +BigCodeBench/341 +BigCodeBench/342 +BigCodeBench/343 +BigCodeBench/344 +BigCodeBench/345 +BigCodeBench/346 +BigCodeBench/347 +BigCodeBench/348 +BigCodeBench/349 +BigCodeBench/350 +BigCodeBench/351 +BigCodeBench/352 +BigCodeBench/353 +BigCodeBench/354 +BigCodeBench/355 +BigCodeBench/356 +BigCodeBench/357 +BigCodeBench/358 +BigCodeBench/359 +BigCodeBench/360 +BigCodeBench/361 +BigCodeBench/362 +BigCodeBench/363 +BigCodeBench/364 +BigCodeBench/365 +BigCodeBench/366 +BigCodeBench/367 +BigCodeBench/368 +BigCodeBench/369 +BigCodeBench/370 +BigCodeBench/371 +BigCodeBench/372 +BigCodeBench/373 +BigCodeBench/374 +BigCodeBench/375 +BigCodeBench/377 +BigCodeBench/378 +BigCodeBench/379 +BigCodeBench/380 +BigCodeBench/381 +BigCodeBench/382 +BigCodeBench/383 +BigCodeBench/384 +BigCodeBench/385 +BigCodeBench/386 +BigCodeBench/387 +BigCodeBench/388 +BigCodeBench/389 +BigCodeBench/391 +BigCodeBench/392 +BigCodeBench/393 +BigCodeBench/394 +BigCodeBench/395 +BigCodeBench/396 +BigCodeBench/398 +BigCodeBench/399 +BigCodeBench/400 +BigCodeBench/403 +BigCodeBench/404 +BigCodeBench/405 +BigCodeBench/406 +BigCodeBench/407 +BigCodeBench/408 +BigCodeBench/409 +BigCodeBench/410 +BigCodeBench/411 +BigCodeBench/412 +BigCodeBench/414 +BigCodeBench/415 +BigCodeBench/416 +BigCodeBench/417 +BigCodeBench/418 +BigCodeBench/419 +BigCodeBench/420 +BigCodeBench/422 +BigCodeBench/423 +BigCodeBench/424 +BigCodeBench/425 +BigCodeBench/426 +BigCodeBench/427 +BigCodeBench/428 +BigCodeBench/429 +BigCodeBench/430 +BigCodeBench/431 +BigCodeBench/432 +BigCodeBench/433 +BigCodeBench/434 +BigCodeBench/435 +BigCodeBench/436 +BigCodeBench/437 +BigCodeBench/438 +BigCodeBench/439 +BigCodeBench/440 +BigCodeBench/441 +BigCodeBench/442 +BigCodeBench/443 +BigCodeBench/444 +BigCodeBench/445 +BigCodeBench/446 +BigCodeBench/447 +BigCodeBench/448 +BigCodeBench/449 +BigCodeBench/450 +BigCodeBench/451 +BigCodeBench/452 +BigCodeBench/453 +BigCodeBench/454 +BigCodeBench/455 +BigCodeBench/456 +BigCodeBench/457 +BigCodeBench/458 +BigCodeBench/459 +BigCodeBench/460 +BigCodeBench/461 +BigCodeBench/462 +BigCodeBench/463 +BigCodeBench/464 +BigCodeBench/465 +BigCodeBench/466 +BigCodeBench/467 +BigCodeBench/468 +BigCodeBench/469 +BigCodeBench/470 +BigCodeBench/471 +BigCodeBench/472 +BigCodeBench/473 +BigCodeBench/474 +BigCodeBench/475 +BigCodeBench/476 +BigCodeBench/477 +BigCodeBench/478 +BigCodeBench/479 +BigCodeBench/480 +BigCodeBench/481 +BigCodeBench/482 +BigCodeBench/483 +BigCodeBench/484 +BigCodeBench/485 +BigCodeBench/486 +BigCodeBench/487 +BigCodeBench/488 +BigCodeBench/489 +BigCodeBench/491 +BigCodeBench/492 +BigCodeBench/493 +BigCodeBench/494 +BigCodeBench/495 +BigCodeBench/496 +BigCodeBench/497 +BigCodeBench/499 +BigCodeBench/500 +BigCodeBench/501 +BigCodeBench/502 +BigCodeBench/503 +BigCodeBench/504 +BigCodeBench/505 +BigCodeBench/506 +BigCodeBench/507 +BigCodeBench/508 +BigCodeBench/509 +BigCodeBench/510 +BigCodeBench/511 +BigCodeBench/512 +BigCodeBench/513 +BigCodeBench/514 +BigCodeBench/515 +BigCodeBench/516 +BigCodeBench/517 +BigCodeBench/518 +BigCodeBench/519 +BigCodeBench/520 +BigCodeBench/521 +BigCodeBench/522 +BigCodeBench/523 +BigCodeBench/524 +BigCodeBench/525 +BigCodeBench/526 +BigCodeBench/527 +BigCodeBench/528 +BigCodeBench/529 +BigCodeBench/530 +BigCodeBench/531 +BigCodeBench/532 +BigCodeBench/533 +BigCodeBench/534 +BigCodeBench/535 +BigCodeBench/536 +BigCodeBench/537 +BigCodeBench/538 +BigCodeBench/539 +BigCodeBench/540 +BigCodeBench/541 +BigCodeBench/542 +BigCodeBench/543 +BigCodeBench/544 +BigCodeBench/545 +BigCodeBench/546 +BigCodeBench/547 +BigCodeBench/548 +BigCodeBench/549 +BigCodeBench/550 +BigCodeBench/551 +BigCodeBench/552 +BigCodeBench/553 +BigCodeBench/554 +BigCodeBench/555 +BigCodeBench/556 +BigCodeBench/557 +BigCodeBench/558 +BigCodeBench/559 +BigCodeBench/560 +BigCodeBench/561 +BigCodeBench/562 +BigCodeBench/563 +BigCodeBench/564 +BigCodeBench/565 +BigCodeBench/566 +BigCodeBench/567 +BigCodeBench/568 +BigCodeBench/569 +BigCodeBench/570 From b2a1b6fc1be8ee81ec32ff541442d3866b986fb3 Mon Sep 17 00:00:00 2001 From: imamnurby Date: Thu, 11 Jul 2024 12:40:00 +0800 Subject: [PATCH 02/14] filter instances that use network, add `offline` param in the function --- bigcodebench/data/bigcodebench.py | 149 +++++++++++++++--------------- 1 file changed, 77 insertions(+), 72 deletions(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index d7fea87..a7bc6d0 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -1,72 +1,77 @@ -import hashlib -import json -import os -from typing import Dict - -from bigcodebench.data.utils import ( - CACHE_DIR, - completeness_check, - get_dataset_metadata, - make_cache, - stream_jsonl, -) -from datasets import load_dataset - -BIGCODEBENCH_OVERRIDE_PATH = os.environ.get("BIGCODEBENCH_OVERRIDE_PATH", None) -BIGCODEBENCH_HF = "bigcode/bigcodebench" -BIGCODEBENCH_VERSION = "v0.1.0_hf" - -def _ready_bigcodebench_path(mini=False, noextreme=False, version="default") -> str: - if BIGCODEBENCH_OVERRIDE_PATH: - return BIGCODEBENCH_OVERRIDE_PATH - - version = BIGCODEBENCH_VERSION if version == "default" else version - url, path = get_dataset_metadata( - "BigCodeBench", BIGCODEBENCH_VERSION, mini, noextreme - ) - - try: - dataset = load_dataset(BIGCODEBENCH_HF, split=BIGCODEBENCH_VERSION) - make_cache(url, dataset, path) - except: - if os.path.exists(path): - os.remove(path) - make_cache(url, None, path, gh=True) - - return path - - -def get_bigcodebench( - err_incomplete=True, mini=False, noextreme=False, version="default" - ) -> Dict[str, Dict]: - """Get BigCodeBench from BigCode's github repo and return as a list of parsed dicts. - - Returns: - List[Dict[str, str]]: List of dicts with keys "complete_prompt", "instruct_prompt", "canonical_solution", "test", "entry_point" - - Notes: - "task_id" is the identifier string for the task. - "complete_prompt" is the prompt to be used for BigCodeBench-Complete. - "instruct_prompt" is the prompt to be used for BigCodeBench-Instruct. - "canonical_solution" is the ground-truth implementation - "test" is the `unittest.TestCase` class. - "entry_point" is the name of the function. - """ - # Check if open eval file exists in CACHE_DIR - data_path = _ready_bigcodebench_path( - mini=mini, noextreme=noextreme, version=version - ) - data = {task["task_id"]: task for task in stream_jsonl(data_path)} - if err_incomplete: - completeness_check("BigCodeBench", data) - return data - -def get_bigcodebench_hash(mini=False, noextreme=False, version="default") -> str: - """Get the hash of BigCodeBench. - Returns: - str: The hash of BigCodeBench - """ - data_path = _ready_bigcodebench_path(mini, noextreme, version="default") - with open(data_path, "rb") as f: - data = f.read() - return hashlib.md5(data).hexdigest() +import hashlib +import json +import os +from typing import Dict + +from bigcodebench.data.utils import ( + CACHE_DIR, + completeness_check, + get_dataset_metadata, + make_cache, + stream_jsonl, +) +from datasets import load_dataset + +BIGCODEBENCH_OVERRIDE_PATH = os.environ.get("BIGCODEBENCH_OVERRIDE_PATH", None) +BIGCODEBENCH_HF = "bigcode/bigcodebench" +BIGCODEBENCH_VERSION = "v0.1.0_hf" + +def _ready_bigcodebench_path(mini=False, noextreme=False, version="default", offline=False) -> str: + if BIGCODEBENCH_OVERRIDE_PATH: + return BIGCODEBENCH_OVERRIDE_PATH + + version = BIGCODEBENCH_VERSION if version == "default" else version + url, path = get_dataset_metadata( + "BigCodeBench", BIGCODEBENCH_VERSION, mini, noextreme + ) + + try: + dataset = load_dataset(BIGCODEBENCH_HF, split=BIGCODEBENCH_VERSION) + if offline: + with open("network-free-set.txt", "r") as f: + included_ids = f.read() + included_ids = included_ids.split("\n") + dataset = dataset.filter(lambda instance: instance["task_id"] in included_ids) + make_cache(url, dataset, path) + except: + if os.path.exists(path): + os.remove(path) + make_cache(url, None, path, gh=True) + + return path + + +def get_bigcodebench( + err_incomplete=True, mini=False, noextreme=False, version="default", offline=False + ) -> Dict[str, Dict]: + """Get BigCodeBench from BigCode's github repo and return as a list of parsed dicts. + + Returns: + List[Dict[str, str]]: List of dicts with keys "complete_prompt", "instruct_prompt", "canonical_solution", "test", "entry_point" + + Notes: + "task_id" is the identifier string for the task. + "complete_prompt" is the prompt to be used for BigCodeBench-Complete. + "instruct_prompt" is the prompt to be used for BigCodeBench-Instruct. + "canonical_solution" is the ground-truth implementation + "test" is the `unittest.TestCase` class. + "entry_point" is the name of the function. + """ + # Check if open eval file exists in CACHE_DIR + data_path = _ready_bigcodebench_path( + mini=mini, noextreme=noextreme, version=version, offline=offline + ) + data = {task["task_id"]: task for task in stream_jsonl(data_path)} + if err_incomplete: + completeness_check("BigCodeBench", data) + return data + +def get_bigcodebench_hash(mini=False, noextreme=False, version="default", offline=False) -> str: + """Get the hash of BigCodeBench. + Returns: + str: The hash of BigCodeBench + """ + data_path = _ready_bigcodebench_path(mini, noextreme, version="default", offline) + with open(data_path, "rb") as f: + data = f.read() + return hashlib.md5(data).hexdigest() From 121a1bb76f76d20fe179311608a1631f6a50f918 Mon Sep 17 00:00:00 2001 From: imamnurby Date: Thu, 11 Jul 2024 12:40:45 +0800 Subject: [PATCH 03/14] fix unused version argument in get_bigcodebench_hash --- bigcodebench/data/bigcodebench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index a7bc6d0..049a758 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -71,7 +71,7 @@ def get_bigcodebench_hash(mini=False, noextreme=False, version="default", offlin Returns: str: The hash of BigCodeBench """ - data_path = _ready_bigcodebench_path(mini, noextreme, version="default", offline) + data_path = _ready_bigcodebench_path(mini, noextreme, version, offline) with open(data_path, "rb") as f: data = f.read() return hashlib.md5(data).hexdigest() From 73c0b4f750652ea4cb086eb39596b6d6f0dc3867 Mon Sep 17 00:00:00 2001 From: imamnurby Date: Thu, 11 Jul 2024 14:44:37 +0800 Subject: [PATCH 04/14] move the filtering inside make_cache to handle both when `gh` is true and false --- bigcodebench/data/bigcodebench.py | 5 ----- bigcodebench/data/utils.py | 21 +++++++++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index 049a758..3e052b5 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -27,11 +27,6 @@ def _ready_bigcodebench_path(mini=False, noextreme=False, version="default", off try: dataset = load_dataset(BIGCODEBENCH_HF, split=BIGCODEBENCH_VERSION) - if offline: - with open("network-free-set.txt", "r") as f: - included_ids = f.read() - included_ids = included_ids.split("\n") - dataset = dataset.filter(lambda instance: instance["task_id"] in included_ids) make_cache(url, dataset, path) except: if os.path.exists(path): diff --git a/bigcodebench/data/utils.py b/bigcodebench/data/utils.py index 4d0a7ee..0a5848f 100644 --- a/bigcodebench/data/utils.py +++ b/bigcodebench/data/utils.py @@ -24,8 +24,14 @@ def get_dataset_metadata(name: str, version: str, mini: bool, noextreme: bool = return url, cache_path -def make_cache(gzip_url, hf_data, cache_path, gh=False): +def make_cache(gzip_url, hf_data, cache_path, gh=False, offline=False): # Check if open eval file exists in CACHE_DIR + + if offline: + with open("network-free-set.txt", "r") as f: + included_ids = f.read() + included_ids = included_ids.split("\n") + if not os.path.exists(cache_path): if gh: @@ -38,14 +44,25 @@ def make_cache(gzip_url, hf_data, cache_path, gh=False): with gzip.open(gz_path, "rb") as f: data = f.read().decode("utf-8") + # If offline, then parse the json then check the task_id + if offline: + json_data = [json.loads(line) for line in data.split('\n') if line] + json_data = [item for item in json_data if item.get("task_id") in included_ids] + # create CACHE_DIR if not exists if not os.path.exists(CACHE_DIR): os.makedirs(CACHE_DIR) # Write the original open eval file to CACHE_DIR with open(cache_path, "w") as f: - f.write(data) + if offline: + for item in json_data: + f.write(json.dumps(item) + '\n') + else: + f.write(data) else: + if offline: + hf_data = hf_data.filter(lambda instance: instance["task_id"] in included_ids) hf_data.to_json(cache_path) From c7e7c9034343b470fc6d0e3caff9736d272ec88a Mon Sep 17 00:00:00 2001 From: imamnurby Date: Thu, 11 Jul 2024 14:46:50 +0800 Subject: [PATCH 05/14] add `offline` argument when calling make_cache --- bigcodebench/data/bigcodebench.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/data/bigcodebench.py b/bigcodebench/data/bigcodebench.py index 3e052b5..6a6ee81 100644 --- a/bigcodebench/data/bigcodebench.py +++ b/bigcodebench/data/bigcodebench.py @@ -27,11 +27,11 @@ def _ready_bigcodebench_path(mini=False, noextreme=False, version="default", off try: dataset = load_dataset(BIGCODEBENCH_HF, split=BIGCODEBENCH_VERSION) - make_cache(url, dataset, path) + make_cache(url, dataset, path, gh=False, offline=offline) except: if os.path.exists(path): os.remove(path) - make_cache(url, None, path, gh=True) + make_cache(url, None, path, gh=True, offline=offline) return path From dc960a02d8977254e09d6cf4720e0bf70af135e9 Mon Sep 17 00:00:00 2001 From: imamnurby Date: Thu, 11 Jul 2024 14:54:39 +0800 Subject: [PATCH 06/14] add `--offline` in the parser argument, add `offline` in the codegen param and get_bigcodebench invocation --- bigcodebench/generate.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bigcodebench/generate.py b/bigcodebench/generate.py index ca21560..3e3f098 100644 --- a/bigcodebench/generate.py +++ b/bigcodebench/generate.py @@ -22,6 +22,7 @@ def codegen( n_samples=1, id_range=None, resume=True, + offline=False, ): with Progress( TextColumn(f"BigCodeBench--{subset} •" + "[progress.percentage]{task.percentage:>3.0f}%"), @@ -31,7 +32,7 @@ def codegen( TimeElapsedColumn(), ) as p: - dataset = get_bigcodebench() + dataset = get_bigcodebench(offline=offline) if model.is_direct_completion() and subset == "instruct": raise Exception("Base model does not support direct completion for instruct tasks") @@ -119,6 +120,7 @@ def main(): parser.add_argument("--tp", default=1, type=int) parser.add_argument("--trust_remote_code", action="store_true") parser.add_argument("--tokenizer_name", default=None, type=str) + parser.add_argument("--offline", action="store_true") args = parser.parse_args() @@ -164,7 +166,8 @@ def main(): strip_newlines=args.strip_newlines, n_samples=args.n_samples, resume=args.resume, - id_range=args.id_range + id_range=args.id_range, + offline=args.offline ) From b03fb74ef5548631417e3ec50185580a2a2d712d Mon Sep 17 00:00:00 2001 From: imamnurby Date: Thu, 11 Jul 2024 14:59:04 +0800 Subject: [PATCH 07/14] add `--offline` argument in the parser --- bigcodebench/evaluate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 6805253..09b6705 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -322,6 +322,7 @@ def main(): parser.add_argument( "--no-gt", action="store_true", help="Check the groundtruth" ) + parser.add_argument("--offline", action="store_true") args = parser.parse_args() evaluate(args) From 88c8303bb99ac507dda49c84c094c5ce15007781 Mon Sep 17 00:00:00 2001 From: imamnurby Date: Thu, 11 Jul 2024 15:00:09 +0800 Subject: [PATCH 08/14] add offline argument when invoking get_bigcodebench --- bigcodebench/evaluate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py index 09b6705..c868d26 100644 --- a/bigcodebench/evaluate.py +++ b/bigcodebench/evaluate.py @@ -124,8 +124,8 @@ def evaluate(flags): assert flags.samples.endswith(".jsonl") result_path = flags.samples.replace(".jsonl", "_eval_results.json") - problems = get_bigcodebench() - dataset_hash = get_bigcodebench_hash() + problems = get_bigcodebench(offline=flags.offline) + dataset_hash = get_bigcodebench_hash(offline=flags.offline) if not flags.no_gt: expected_time = get_groundtruth(n_workers, problems, dataset_hash, flags.check_gt_only, flags.max_as_limit, flags.max_data_limit, flags.max_stack_limit) From ee0a116bc1ff1d9fb2d00fd55c1b126c015b0b4c Mon Sep 17 00:00:00 2001 From: imamnurby Date: Sat, 13 Jul 2024 23:32:14 +0800 Subject: [PATCH 09/14] move ids to constant.py --- bigcodebench/data/constant.py | 520 +++++++++++++++++++++++++++++ bigcodebench/data/create_ids.ipynb | 0 2 files changed, 520 insertions(+) create mode 100644 bigcodebench/data/constant.py create mode 100644 bigcodebench/data/create_ids.ipynb diff --git a/bigcodebench/data/constant.py b/bigcodebench/data/constant.py new file mode 100644 index 0000000..3f9e105 --- /dev/null +++ b/bigcodebench/data/constant.py @@ -0,0 +1,520 @@ + +OFFLINE_IDS = ( + 'BigCodeBench/0', + 'BigCodeBench/1', + 'BigCodeBench/2', + 'BigCodeBench/3', + 'BigCodeBench/4', + 'BigCodeBench/5', + 'BigCodeBench/6', + 'BigCodeBench/7', + 'BigCodeBench/8', + 'BigCodeBench/9', + 'BigCodeBench/10', + 'BigCodeBench/11', + 'BigCodeBench/12', + 'BigCodeBench/14', + 'BigCodeBench/15', + 'BigCodeBench/16', + 'BigCodeBench/17', + 'BigCodeBench/18', + 'BigCodeBench/19', + 'BigCodeBench/20', + 'BigCodeBench/21', + 'BigCodeBench/22', + 'BigCodeBench/23', + 'BigCodeBench/24', + 'BigCodeBench/25', + 'BigCodeBench/26', + 'BigCodeBench/27', + 'BigCodeBench/29', + 'BigCodeBench/30', + 'BigCodeBench/31', + 'BigCodeBench/33', + 'BigCodeBench/34', + 'BigCodeBench/35', + 'BigCodeBench/36', + 'BigCodeBench/37', + 'BigCodeBench/38', + 'BigCodeBench/39', + 'BigCodeBench/40', + 'BigCodeBench/41', + 'BigCodeBench/42', + 'BigCodeBench/43', + 'BigCodeBench/44', + 'BigCodeBench/45', + 'BigCodeBench/46', + 'BigCodeBench/47', + 'BigCodeBench/48', + 'BigCodeBench/49', + 'BigCodeBench/50', + 'BigCodeBench/51', + 'BigCodeBench/52', + 'BigCodeBench/53', + 'BigCodeBench/54', + 'BigCodeBench/55', + 'BigCodeBench/56', + 'BigCodeBench/57', + 'BigCodeBench/58', + 'BigCodeBench/60', + 'BigCodeBench/61', + 'BigCodeBench/62', + 'BigCodeBench/63', + 'BigCodeBench/64', + 'BigCodeBench/65', + 'BigCodeBench/66', + 'BigCodeBench/67', + 'BigCodeBench/68', + 'BigCodeBench/69', + 'BigCodeBench/70', + 'BigCodeBench/71', + 'BigCodeBench/72', + 'BigCodeBench/73', + 'BigCodeBench/75', + 'BigCodeBench/84', + 'BigCodeBench/85', + 'BigCodeBench/86', + 'BigCodeBench/87', + 'BigCodeBench/88', + 'BigCodeBench/89', + 'BigCodeBench/90', + 'BigCodeBench/91', + 'BigCodeBench/92', + 'BigCodeBench/93', + 'BigCodeBench/94', + 'BigCodeBench/95', + 'BigCodeBench/96', + 'BigCodeBench/97', + 'BigCodeBench/98', + 'BigCodeBench/99', + 'BigCodeBench/100', + 'BigCodeBench/102', + 'BigCodeBench/103', + 'BigCodeBench/104', + 'BigCodeBench/105', + 'BigCodeBench/106', + 'BigCodeBench/107', + 'BigCodeBench/108', + 'BigCodeBench/109', + 'BigCodeBench/110', + 'BigCodeBench/111', + 'BigCodeBench/112', + 'BigCodeBench/113', + 'BigCodeBench/114', + 'BigCodeBench/115', + 'BigCodeBench/116', + 'BigCodeBench/117', + 'BigCodeBench/118', + 'BigCodeBench/119', + 'BigCodeBench/120', + 'BigCodeBench/121', + 'BigCodeBench/122', + 'BigCodeBench/123', + 'BigCodeBench/124', + 'BigCodeBench/125', + 'BigCodeBench/126', + 'BigCodeBench/127', + 'BigCodeBench/128', + 'BigCodeBench/130', + 'BigCodeBench/131', + 'BigCodeBench/132', + 'BigCodeBench/133', + 'BigCodeBench/134', + 'BigCodeBench/135', + 'BigCodeBench/136', + 'BigCodeBench/137', + 'BigCodeBench/138', + 'BigCodeBench/139', + 'BigCodeBench/140', + 'BigCodeBench/141', + 'BigCodeBench/142', + 'BigCodeBench/143', + 'BigCodeBench/148', + 'BigCodeBench/149', + 'BigCodeBench/150', + 'BigCodeBench/151', + 'BigCodeBench/152', + 'BigCodeBench/153', + 'BigCodeBench/155', + 'BigCodeBench/156', + 'BigCodeBench/157', + 'BigCodeBench/159', + 'BigCodeBench/160', + 'BigCodeBench/161', + 'BigCodeBench/162', + 'BigCodeBench/163', + 'BigCodeBench/164', + 'BigCodeBench/165', + 'BigCodeBench/166', + 'BigCodeBench/167', + 'BigCodeBench/168', + 'BigCodeBench/169', + 'BigCodeBench/171', + 'BigCodeBench/172', + 'BigCodeBench/173', + 'BigCodeBench/174', + 'BigCodeBench/175', + 'BigCodeBench/177', + 'BigCodeBench/178', + 'BigCodeBench/179', + 'BigCodeBench/180', + 'BigCodeBench/182', + 'BigCodeBench/184', + 'BigCodeBench/187', + 'BigCodeBench/190', + 'BigCodeBench/191', + 'BigCodeBench/193', + 'BigCodeBench/194', + 'BigCodeBench/196', + 'BigCodeBench/197', + 'BigCodeBench/198', + 'BigCodeBench/199', + 'BigCodeBench/200', + 'BigCodeBench/201', + 'BigCodeBench/202', + 'BigCodeBench/204', + 'BigCodeBench/205', + 'BigCodeBench/206', + 'BigCodeBench/208', + 'BigCodeBench/209', + 'BigCodeBench/210', + 'BigCodeBench/212', + 'BigCodeBench/213', + 'BigCodeBench/214', + 'BigCodeBench/216', + 'BigCodeBench/217', + 'BigCodeBench/218', + 'BigCodeBench/219', + 'BigCodeBench/220', + 'BigCodeBench/221', + 'BigCodeBench/222', + 'BigCodeBench/223', + 'BigCodeBench/224', + 'BigCodeBench/225', + 'BigCodeBench/226', + 'BigCodeBench/227', + 'BigCodeBench/228', + 'BigCodeBench/229', + 'BigCodeBench/230', + 'BigCodeBench/231', + 'BigCodeBench/232', + 'BigCodeBench/233', + 'BigCodeBench/234', + 'BigCodeBench/235', + 'BigCodeBench/236', + 'BigCodeBench/237', + 'BigCodeBench/238', + 'BigCodeBench/239', + 'BigCodeBench/240', + 'BigCodeBench/241', + 'BigCodeBench/242', + 'BigCodeBench/243', + 'BigCodeBench/244', + 'BigCodeBench/245', + 'BigCodeBench/246', + 'BigCodeBench/247', + 'BigCodeBench/248', + 'BigCodeBench/249', + 'BigCodeBench/250', + 'BigCodeBench/251', + 'BigCodeBench/252', + 'BigCodeBench/253', + 'BigCodeBench/254', + 'BigCodeBench/255', + 'BigCodeBench/256', + 'BigCodeBench/257', + 'BigCodeBench/258', + 'BigCodeBench/259', + 'BigCodeBench/260', + 'BigCodeBench/261', + 'BigCodeBench/262', + 'BigCodeBench/263', + 'BigCodeBench/264', + 'BigCodeBench/265', + 'BigCodeBench/266', + 'BigCodeBench/267', + 'BigCodeBench/268', + 'BigCodeBench/269', + 'BigCodeBench/270', + 'BigCodeBench/271', + 'BigCodeBench/275', + 'BigCodeBench/276', + 'BigCodeBench/277', + 'BigCodeBench/278', + 'BigCodeBench/279', + 'BigCodeBench/280', + 'BigCodeBench/281', + 'BigCodeBench/282', + 'BigCodeBench/283', + 'BigCodeBench/286', + 'BigCodeBench/287', + 'BigCodeBench/288', + 'BigCodeBench/289', + 'BigCodeBench/291', + 'BigCodeBench/292', + 'BigCodeBench/293', + 'BigCodeBench/294', + 'BigCodeBench/295', + 'BigCodeBench/296', + 'BigCodeBench/297', + 'BigCodeBench/298', + 'BigCodeBench/299', + 'BigCodeBench/300', + 'BigCodeBench/301', + 'BigCodeBench/302', + 'BigCodeBench/303', + 'BigCodeBench/304', + 'BigCodeBench/305', + 'BigCodeBench/306', + 'BigCodeBench/307', + 'BigCodeBench/308', + 'BigCodeBench/309', + 'BigCodeBench/310', + 'BigCodeBench/311', + 'BigCodeBench/312', + 'BigCodeBench/313', + 'BigCodeBench/316', + 'BigCodeBench/317', + 'BigCodeBench/318', + 'BigCodeBench/319', + 'BigCodeBench/320', + 'BigCodeBench/321', + 'BigCodeBench/322', + 'BigCodeBench/323', + 'BigCodeBench/324', + 'BigCodeBench/325', + 'BigCodeBench/326', + 'BigCodeBench/327', + 'BigCodeBench/328', + 'BigCodeBench/329', + 'BigCodeBench/330', + 'BigCodeBench/331', + 'BigCodeBench/333', + 'BigCodeBench/334', + 'BigCodeBench/335', + 'BigCodeBench/336', + 'BigCodeBench/337', + 'BigCodeBench/338', + 'BigCodeBench/340', + 'BigCodeBench/341', + 'BigCodeBench/342', + 'BigCodeBench/343', + 'BigCodeBench/344', + 'BigCodeBench/345', + 'BigCodeBench/346', + 'BigCodeBench/347', + 'BigCodeBench/348', + 'BigCodeBench/349', + 'BigCodeBench/350', + 'BigCodeBench/351', + 'BigCodeBench/352', + 'BigCodeBench/353', + 'BigCodeBench/354', + 'BigCodeBench/355', + 'BigCodeBench/356', + 'BigCodeBench/357', + 'BigCodeBench/358', + 'BigCodeBench/359', + 'BigCodeBench/360', + 'BigCodeBench/361', + 'BigCodeBench/362', + 'BigCodeBench/363', + 'BigCodeBench/364', + 'BigCodeBench/365', + 'BigCodeBench/366', + 'BigCodeBench/367', + 'BigCodeBench/368', + 'BigCodeBench/369', + 'BigCodeBench/370', + 'BigCodeBench/371', + 'BigCodeBench/372', + 'BigCodeBench/373', + 'BigCodeBench/374', + 'BigCodeBench/375', + 'BigCodeBench/377', + 'BigCodeBench/378', + 'BigCodeBench/379', + 'BigCodeBench/380', + 'BigCodeBench/381', + 'BigCodeBench/382', + 'BigCodeBench/383', + 'BigCodeBench/384', + 'BigCodeBench/385', + 'BigCodeBench/386', + 'BigCodeBench/387', + 'BigCodeBench/388', + 'BigCodeBench/389', + 'BigCodeBench/391', + 'BigCodeBench/392', + 'BigCodeBench/393', + 'BigCodeBench/394', + 'BigCodeBench/395', + 'BigCodeBench/396', + 'BigCodeBench/398', + 'BigCodeBench/399', + 'BigCodeBench/400', + 'BigCodeBench/403', + 'BigCodeBench/404', + 'BigCodeBench/405', + 'BigCodeBench/406', + 'BigCodeBench/407', + 'BigCodeBench/408', + 'BigCodeBench/409', + 'BigCodeBench/410', + 'BigCodeBench/411', + 'BigCodeBench/412', + 'BigCodeBench/414', + 'BigCodeBench/415', + 'BigCodeBench/416', + 'BigCodeBench/417', + 'BigCodeBench/418', + 'BigCodeBench/419', + 'BigCodeBench/420', + 'BigCodeBench/422', + 'BigCodeBench/423', + 'BigCodeBench/424', + 'BigCodeBench/425', + 'BigCodeBench/426', + 'BigCodeBench/427', + 'BigCodeBench/428', + 'BigCodeBench/429', + 'BigCodeBench/430', + 'BigCodeBench/431', + 'BigCodeBench/432', + 'BigCodeBench/433', + 'BigCodeBench/434', + 'BigCodeBench/435', + 'BigCodeBench/436', + 'BigCodeBench/437', + 'BigCodeBench/438', + 'BigCodeBench/439', + 'BigCodeBench/440', + 'BigCodeBench/441', + 'BigCodeBench/442', + 'BigCodeBench/443', + 'BigCodeBench/444', + 'BigCodeBench/445', + 'BigCodeBench/446', + 'BigCodeBench/447', + 'BigCodeBench/448', + 'BigCodeBench/449', + 'BigCodeBench/450', + 'BigCodeBench/451', + 'BigCodeBench/452', + 'BigCodeBench/453', + 'BigCodeBench/454', + 'BigCodeBench/455', + 'BigCodeBench/456', + 'BigCodeBench/457', + 'BigCodeBench/458', + 'BigCodeBench/459', + 'BigCodeBench/460', + 'BigCodeBench/461', + 'BigCodeBench/462', + 'BigCodeBench/463', + 'BigCodeBench/464', + 'BigCodeBench/465', + 'BigCodeBench/466', + 'BigCodeBench/467', + 'BigCodeBench/468', + 'BigCodeBench/469', + 'BigCodeBench/470', + 'BigCodeBench/471', + 'BigCodeBench/472', + 'BigCodeBench/473', + 'BigCodeBench/474', + 'BigCodeBench/475', + 'BigCodeBench/476', + 'BigCodeBench/477', + 'BigCodeBench/478', + 'BigCodeBench/479', + 'BigCodeBench/480', + 'BigCodeBench/481', + 'BigCodeBench/482', + 'BigCodeBench/483', + 'BigCodeBench/484', + 'BigCodeBench/485', + 'BigCodeBench/486', + 'BigCodeBench/487', + 'BigCodeBench/488', + 'BigCodeBench/489', + 'BigCodeBench/491', + 'BigCodeBench/492', + 'BigCodeBench/493', + 'BigCodeBench/494', + 'BigCodeBench/495', + 'BigCodeBench/496', + 'BigCodeBench/497', + 'BigCodeBench/499', + 'BigCodeBench/500', + 'BigCodeBench/501', + 'BigCodeBench/502', + 'BigCodeBench/503', + 'BigCodeBench/504', + 'BigCodeBench/505', + 'BigCodeBench/506', + 'BigCodeBench/507', + 'BigCodeBench/508', + 'BigCodeBench/509', + 'BigCodeBench/510', + 'BigCodeBench/511', + 'BigCodeBench/512', + 'BigCodeBench/513', + 'BigCodeBench/514', + 'BigCodeBench/515', + 'BigCodeBench/516', + 'BigCodeBench/517', + 'BigCodeBench/518', + 'BigCodeBench/519', + 'BigCodeBench/520', + 'BigCodeBench/521', + 'BigCodeBench/522', + 'BigCodeBench/523', + 'BigCodeBench/524', + 'BigCodeBench/525', + 'BigCodeBench/526', + 'BigCodeBench/527', + 'BigCodeBench/528', + 'BigCodeBench/529', + 'BigCodeBench/530', + 'BigCodeBench/531', + 'BigCodeBench/532', + 'BigCodeBench/533', + 'BigCodeBench/534', + 'BigCodeBench/535', + 'BigCodeBench/536', + 'BigCodeBench/537', + 'BigCodeBench/538', + 'BigCodeBench/539', + 'BigCodeBench/540', + 'BigCodeBench/541', + 'BigCodeBench/542', + 'BigCodeBench/543', + 'BigCodeBench/544', + 'BigCodeBench/545', + 'BigCodeBench/546', + 'BigCodeBench/547', + 'BigCodeBench/548', + 'BigCodeBench/549', + 'BigCodeBench/550', + 'BigCodeBench/551', + 'BigCodeBench/552', + 'BigCodeBench/553', + 'BigCodeBench/554', + 'BigCodeBench/555', + 'BigCodeBench/556', + 'BigCodeBench/557', + 'BigCodeBench/558', + 'BigCodeBench/559', + 'BigCodeBench/560', + 'BigCodeBench/561', + 'BigCodeBench/562', + 'BigCodeBench/563', + 'BigCodeBench/564', + 'BigCodeBench/565', + 'BigCodeBench/566', + 'BigCodeBench/567', + 'BigCodeBench/568', + 'BigCodeBench/569', + 'BigCodeBench/570' +) \ No newline at end of file diff --git a/bigcodebench/data/create_ids.ipynb b/bigcodebench/data/create_ids.ipynb new file mode 100644 index 0000000..e69de29 From f61cc7733e94d6d99aeea598e218f803044ab14c Mon Sep 17 00:00:00 2001 From: imamnurby Date: Sat, 13 Jul 2024 23:32:48 +0800 Subject: [PATCH 10/14] remove network-free-set.txt --- bigcodebench/data/network-free-set.txt | 517 ------------------------- 1 file changed, 517 deletions(-) delete mode 100644 bigcodebench/data/network-free-set.txt diff --git a/bigcodebench/data/network-free-set.txt b/bigcodebench/data/network-free-set.txt deleted file mode 100644 index 2d43724..0000000 --- a/bigcodebench/data/network-free-set.txt +++ /dev/null @@ -1,517 +0,0 @@ -BigCodeBench/0 -BigCodeBench/1 -BigCodeBench/2 -BigCodeBench/3 -BigCodeBench/4 -BigCodeBench/5 -BigCodeBench/6 -BigCodeBench/7 -BigCodeBench/8 -BigCodeBench/9 -BigCodeBench/10 -BigCodeBench/11 -BigCodeBench/12 -BigCodeBench/14 -BigCodeBench/15 -BigCodeBench/16 -BigCodeBench/17 -BigCodeBench/18 -BigCodeBench/19 -BigCodeBench/20 -BigCodeBench/21 -BigCodeBench/22 -BigCodeBench/23 -BigCodeBench/24 -BigCodeBench/25 -BigCodeBench/26 -BigCodeBench/27 -BigCodeBench/29 -BigCodeBench/30 -BigCodeBench/31 -BigCodeBench/33 -BigCodeBench/34 -BigCodeBench/35 -BigCodeBench/36 -BigCodeBench/37 -BigCodeBench/38 -BigCodeBench/39 -BigCodeBench/40 -BigCodeBench/41 -BigCodeBench/42 -BigCodeBench/43 -BigCodeBench/44 -BigCodeBench/45 -BigCodeBench/46 -BigCodeBench/47 -BigCodeBench/48 -BigCodeBench/49 -BigCodeBench/50 -BigCodeBench/51 -BigCodeBench/52 -BigCodeBench/53 -BigCodeBench/54 -BigCodeBench/55 -BigCodeBench/56 -BigCodeBench/57 -BigCodeBench/58 -BigCodeBench/60 -BigCodeBench/61 -BigCodeBench/62 -BigCodeBench/63 -BigCodeBench/64 -BigCodeBench/65 -BigCodeBench/66 -BigCodeBench/67 -BigCodeBench/68 -BigCodeBench/69 -BigCodeBench/70 -BigCodeBench/71 -BigCodeBench/72 -BigCodeBench/73 -BigCodeBench/75 -BigCodeBench/84 -BigCodeBench/85 -BigCodeBench/86 -BigCodeBench/87 -BigCodeBench/88 -BigCodeBench/89 -BigCodeBench/90 -BigCodeBench/91 -BigCodeBench/92 -BigCodeBench/93 -BigCodeBench/94 -BigCodeBench/95 -BigCodeBench/96 -BigCodeBench/97 -BigCodeBench/98 -BigCodeBench/99 -BigCodeBench/100 -BigCodeBench/102 -BigCodeBench/103 -BigCodeBench/104 -BigCodeBench/105 -BigCodeBench/106 -BigCodeBench/107 -BigCodeBench/108 -BigCodeBench/109 -BigCodeBench/110 -BigCodeBench/111 -BigCodeBench/112 -BigCodeBench/113 -BigCodeBench/114 -BigCodeBench/115 -BigCodeBench/116 -BigCodeBench/117 -BigCodeBench/118 -BigCodeBench/119 -BigCodeBench/120 -BigCodeBench/121 -BigCodeBench/122 -BigCodeBench/123 -BigCodeBench/124 -BigCodeBench/125 -BigCodeBench/126 -BigCodeBench/127 -BigCodeBench/128 -BigCodeBench/130 -BigCodeBench/131 -BigCodeBench/132 -BigCodeBench/133 -BigCodeBench/134 -BigCodeBench/135 -BigCodeBench/136 -BigCodeBench/137 -BigCodeBench/138 -BigCodeBench/139 -BigCodeBench/140 -BigCodeBench/141 -BigCodeBench/142 -BigCodeBench/143 -BigCodeBench/148 -BigCodeBench/149 -BigCodeBench/150 -BigCodeBench/151 -BigCodeBench/152 -BigCodeBench/153 -BigCodeBench/155 -BigCodeBench/156 -BigCodeBench/157 -BigCodeBench/159 -BigCodeBench/160 -BigCodeBench/161 -BigCodeBench/162 -BigCodeBench/163 -BigCodeBench/164 -BigCodeBench/165 -BigCodeBench/166 -BigCodeBench/167 -BigCodeBench/168 -BigCodeBench/169 -BigCodeBench/171 -BigCodeBench/172 -BigCodeBench/173 -BigCodeBench/174 -BigCodeBench/175 -BigCodeBench/177 -BigCodeBench/178 -BigCodeBench/179 -BigCodeBench/180 -BigCodeBench/182 -BigCodeBench/184 -BigCodeBench/187 -BigCodeBench/190 -BigCodeBench/191 -BigCodeBench/193 -BigCodeBench/194 -BigCodeBench/196 -BigCodeBench/197 -BigCodeBench/198 -BigCodeBench/199 -BigCodeBench/200 -BigCodeBench/201 -BigCodeBench/202 -BigCodeBench/204 -BigCodeBench/205 -BigCodeBench/206 -BigCodeBench/208 -BigCodeBench/209 -BigCodeBench/210 -BigCodeBench/212 -BigCodeBench/213 -BigCodeBench/214 -BigCodeBench/216 -BigCodeBench/217 -BigCodeBench/218 -BigCodeBench/219 -BigCodeBench/220 -BigCodeBench/221 -BigCodeBench/222 -BigCodeBench/223 -BigCodeBench/224 -BigCodeBench/225 -BigCodeBench/226 -BigCodeBench/227 -BigCodeBench/228 -BigCodeBench/229 -BigCodeBench/230 -BigCodeBench/231 -BigCodeBench/232 -BigCodeBench/233 -BigCodeBench/234 -BigCodeBench/235 -BigCodeBench/236 -BigCodeBench/237 -BigCodeBench/238 -BigCodeBench/239 -BigCodeBench/240 -BigCodeBench/241 -BigCodeBench/242 -BigCodeBench/243 -BigCodeBench/244 -BigCodeBench/245 -BigCodeBench/246 -BigCodeBench/247 -BigCodeBench/248 -BigCodeBench/249 -BigCodeBench/250 -BigCodeBench/251 -BigCodeBench/252 -BigCodeBench/253 -BigCodeBench/254 -BigCodeBench/255 -BigCodeBench/256 -BigCodeBench/257 -BigCodeBench/258 -BigCodeBench/259 -BigCodeBench/260 -BigCodeBench/261 -BigCodeBench/262 -BigCodeBench/263 -BigCodeBench/264 -BigCodeBench/265 -BigCodeBench/266 -BigCodeBench/267 -BigCodeBench/268 -BigCodeBench/269 -BigCodeBench/270 -BigCodeBench/271 -BigCodeBench/275 -BigCodeBench/276 -BigCodeBench/277 -BigCodeBench/278 -BigCodeBench/279 -BigCodeBench/280 -BigCodeBench/281 -BigCodeBench/282 -BigCodeBench/283 -BigCodeBench/286 -BigCodeBench/287 -BigCodeBench/288 -BigCodeBench/289 -BigCodeBench/291 -BigCodeBench/292 -BigCodeBench/293 -BigCodeBench/294 -BigCodeBench/295 -BigCodeBench/296 -BigCodeBench/297 -BigCodeBench/298 -BigCodeBench/299 -BigCodeBench/300 -BigCodeBench/301 -BigCodeBench/302 -BigCodeBench/303 -BigCodeBench/304 -BigCodeBench/305 -BigCodeBench/306 -BigCodeBench/307 -BigCodeBench/308 -BigCodeBench/309 -BigCodeBench/310 -BigCodeBench/311 -BigCodeBench/312 -BigCodeBench/313 -BigCodeBench/316 -BigCodeBench/317 -BigCodeBench/318 -BigCodeBench/319 -BigCodeBench/320 -BigCodeBench/321 -BigCodeBench/322 -BigCodeBench/323 -BigCodeBench/324 -BigCodeBench/325 -BigCodeBench/326 -BigCodeBench/327 -BigCodeBench/328 -BigCodeBench/329 -BigCodeBench/330 -BigCodeBench/331 -BigCodeBench/333 -BigCodeBench/334 -BigCodeBench/335 -BigCodeBench/336 -BigCodeBench/337 -BigCodeBench/338 -BigCodeBench/340 -BigCodeBench/341 -BigCodeBench/342 -BigCodeBench/343 -BigCodeBench/344 -BigCodeBench/345 -BigCodeBench/346 -BigCodeBench/347 -BigCodeBench/348 -BigCodeBench/349 -BigCodeBench/350 -BigCodeBench/351 -BigCodeBench/352 -BigCodeBench/353 -BigCodeBench/354 -BigCodeBench/355 -BigCodeBench/356 -BigCodeBench/357 -BigCodeBench/358 -BigCodeBench/359 -BigCodeBench/360 -BigCodeBench/361 -BigCodeBench/362 -BigCodeBench/363 -BigCodeBench/364 -BigCodeBench/365 -BigCodeBench/366 -BigCodeBench/367 -BigCodeBench/368 -BigCodeBench/369 -BigCodeBench/370 -BigCodeBench/371 -BigCodeBench/372 -BigCodeBench/373 -BigCodeBench/374 -BigCodeBench/375 -BigCodeBench/377 -BigCodeBench/378 -BigCodeBench/379 -BigCodeBench/380 -BigCodeBench/381 -BigCodeBench/382 -BigCodeBench/383 -BigCodeBench/384 -BigCodeBench/385 -BigCodeBench/386 -BigCodeBench/387 -BigCodeBench/388 -BigCodeBench/389 -BigCodeBench/391 -BigCodeBench/392 -BigCodeBench/393 -BigCodeBench/394 -BigCodeBench/395 -BigCodeBench/396 -BigCodeBench/398 -BigCodeBench/399 -BigCodeBench/400 -BigCodeBench/403 -BigCodeBench/404 -BigCodeBench/405 -BigCodeBench/406 -BigCodeBench/407 -BigCodeBench/408 -BigCodeBench/409 -BigCodeBench/410 -BigCodeBench/411 -BigCodeBench/412 -BigCodeBench/414 -BigCodeBench/415 -BigCodeBench/416 -BigCodeBench/417 -BigCodeBench/418 -BigCodeBench/419 -BigCodeBench/420 -BigCodeBench/422 -BigCodeBench/423 -BigCodeBench/424 -BigCodeBench/425 -BigCodeBench/426 -BigCodeBench/427 -BigCodeBench/428 -BigCodeBench/429 -BigCodeBench/430 -BigCodeBench/431 -BigCodeBench/432 -BigCodeBench/433 -BigCodeBench/434 -BigCodeBench/435 -BigCodeBench/436 -BigCodeBench/437 -BigCodeBench/438 -BigCodeBench/439 -BigCodeBench/440 -BigCodeBench/441 -BigCodeBench/442 -BigCodeBench/443 -BigCodeBench/444 -BigCodeBench/445 -BigCodeBench/446 -BigCodeBench/447 -BigCodeBench/448 -BigCodeBench/449 -BigCodeBench/450 -BigCodeBench/451 -BigCodeBench/452 -BigCodeBench/453 -BigCodeBench/454 -BigCodeBench/455 -BigCodeBench/456 -BigCodeBench/457 -BigCodeBench/458 -BigCodeBench/459 -BigCodeBench/460 -BigCodeBench/461 -BigCodeBench/462 -BigCodeBench/463 -BigCodeBench/464 -BigCodeBench/465 -BigCodeBench/466 -BigCodeBench/467 -BigCodeBench/468 -BigCodeBench/469 -BigCodeBench/470 -BigCodeBench/471 -BigCodeBench/472 -BigCodeBench/473 -BigCodeBench/474 -BigCodeBench/475 -BigCodeBench/476 -BigCodeBench/477 -BigCodeBench/478 -BigCodeBench/479 -BigCodeBench/480 -BigCodeBench/481 -BigCodeBench/482 -BigCodeBench/483 -BigCodeBench/484 -BigCodeBench/485 -BigCodeBench/486 -BigCodeBench/487 -BigCodeBench/488 -BigCodeBench/489 -BigCodeBench/491 -BigCodeBench/492 -BigCodeBench/493 -BigCodeBench/494 -BigCodeBench/495 -BigCodeBench/496 -BigCodeBench/497 -BigCodeBench/499 -BigCodeBench/500 -BigCodeBench/501 -BigCodeBench/502 -BigCodeBench/503 -BigCodeBench/504 -BigCodeBench/505 -BigCodeBench/506 -BigCodeBench/507 -BigCodeBench/508 -BigCodeBench/509 -BigCodeBench/510 -BigCodeBench/511 -BigCodeBench/512 -BigCodeBench/513 -BigCodeBench/514 -BigCodeBench/515 -BigCodeBench/516 -BigCodeBench/517 -BigCodeBench/518 -BigCodeBench/519 -BigCodeBench/520 -BigCodeBench/521 -BigCodeBench/522 -BigCodeBench/523 -BigCodeBench/524 -BigCodeBench/525 -BigCodeBench/526 -BigCodeBench/527 -BigCodeBench/528 -BigCodeBench/529 -BigCodeBench/530 -BigCodeBench/531 -BigCodeBench/532 -BigCodeBench/533 -BigCodeBench/534 -BigCodeBench/535 -BigCodeBench/536 -BigCodeBench/537 -BigCodeBench/538 -BigCodeBench/539 -BigCodeBench/540 -BigCodeBench/541 -BigCodeBench/542 -BigCodeBench/543 -BigCodeBench/544 -BigCodeBench/545 -BigCodeBench/546 -BigCodeBench/547 -BigCodeBench/548 -BigCodeBench/549 -BigCodeBench/550 -BigCodeBench/551 -BigCodeBench/552 -BigCodeBench/553 -BigCodeBench/554 -BigCodeBench/555 -BigCodeBench/556 -BigCodeBench/557 -BigCodeBench/558 -BigCodeBench/559 -BigCodeBench/560 -BigCodeBench/561 -BigCodeBench/562 -BigCodeBench/563 -BigCodeBench/564 -BigCodeBench/565 -BigCodeBench/566 -BigCodeBench/567 -BigCodeBench/568 -BigCodeBench/569 -BigCodeBench/570 From b61b23c481505962b0ab0de591ef91b469324382 Mon Sep 17 00:00:00 2001 From: imamnurby Date: Sat, 13 Jul 2024 23:34:40 +0800 Subject: [PATCH 11/14] replace reading ids from the txt file with importing `OFFLINE_IDS` from `constant.py` --- bigcodebench/data/utils.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/bigcodebench/data/utils.py b/bigcodebench/data/utils.py index 0a5848f..91c5db2 100644 --- a/bigcodebench/data/utils.py +++ b/bigcodebench/data/utils.py @@ -8,6 +8,8 @@ import wget from appdirs import user_cache_dir +from .constant import OFFLINE_IDS + CACHE_DIR = user_cache_dir("bigcodebench") @@ -27,11 +29,6 @@ def get_dataset_metadata(name: str, version: str, mini: bool, noextreme: bool = def make_cache(gzip_url, hf_data, cache_path, gh=False, offline=False): # Check if open eval file exists in CACHE_DIR - if offline: - with open("network-free-set.txt", "r") as f: - included_ids = f.read() - included_ids = included_ids.split("\n") - if not os.path.exists(cache_path): if gh: @@ -47,7 +44,7 @@ def make_cache(gzip_url, hf_data, cache_path, gh=False, offline=False): # If offline, then parse the json then check the task_id if offline: json_data = [json.loads(line) for line in data.split('\n') if line] - json_data = [item for item in json_data if item.get("task_id") in included_ids] + json_data = [item for item in json_data if item.get("task_id") in OFFLINE_IDS] # create CACHE_DIR if not exists if not os.path.exists(CACHE_DIR): @@ -62,7 +59,7 @@ def make_cache(gzip_url, hf_data, cache_path, gh=False, offline=False): f.write(data) else: if offline: - hf_data = hf_data.filter(lambda instance: instance["task_id"] in included_ids) + hf_data = hf_data.filter(lambda instance: instance["task_id"] in OFFLINE_IDS) hf_data.to_json(cache_path) From c02f0d01452447675a86ae8c897284539a396b83 Mon Sep 17 00:00:00 2001 From: Terry Zhuo Date: Tue, 16 Jul 2024 12:55:38 +0800 Subject: [PATCH 12/14] Delete bigcodebench/data/create_ids.ipynb --- bigcodebench/data/create_ids.ipynb | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 bigcodebench/data/create_ids.ipynb diff --git a/bigcodebench/data/create_ids.ipynb b/bigcodebench/data/create_ids.ipynb deleted file mode 100644 index e69de29..0000000 From d951246c74084ebc4e7c77f53a4c347bbc92effc Mon Sep 17 00:00:00 2001 From: imamnurby Date: Wed, 17 Jul 2024 20:49:34 +0800 Subject: [PATCH 13/14] remove `BigCodeBench/` --- bigcodebench/data/constant.py | 1034 ++++++++++++++++----------------- 1 file changed, 517 insertions(+), 517 deletions(-) diff --git a/bigcodebench/data/constant.py b/bigcodebench/data/constant.py index 3f9e105..1002409 100644 --- a/bigcodebench/data/constant.py +++ b/bigcodebench/data/constant.py @@ -1,520 +1,520 @@ OFFLINE_IDS = ( - 'BigCodeBench/0', - 'BigCodeBench/1', - 'BigCodeBench/2', - 'BigCodeBench/3', - 'BigCodeBench/4', - 'BigCodeBench/5', - 'BigCodeBench/6', - 'BigCodeBench/7', - 'BigCodeBench/8', - 'BigCodeBench/9', - 'BigCodeBench/10', - 'BigCodeBench/11', - 'BigCodeBench/12', - 'BigCodeBench/14', - 'BigCodeBench/15', - 'BigCodeBench/16', - 'BigCodeBench/17', - 'BigCodeBench/18', - 'BigCodeBench/19', - 'BigCodeBench/20', - 'BigCodeBench/21', - 'BigCodeBench/22', - 'BigCodeBench/23', - 'BigCodeBench/24', - 'BigCodeBench/25', - 'BigCodeBench/26', - 'BigCodeBench/27', - 'BigCodeBench/29', - 'BigCodeBench/30', - 'BigCodeBench/31', - 'BigCodeBench/33', - 'BigCodeBench/34', - 'BigCodeBench/35', - 'BigCodeBench/36', - 'BigCodeBench/37', - 'BigCodeBench/38', - 'BigCodeBench/39', - 'BigCodeBench/40', - 'BigCodeBench/41', - 'BigCodeBench/42', - 'BigCodeBench/43', - 'BigCodeBench/44', - 'BigCodeBench/45', - 'BigCodeBench/46', - 'BigCodeBench/47', - 'BigCodeBench/48', - 'BigCodeBench/49', - 'BigCodeBench/50', - 'BigCodeBench/51', - 'BigCodeBench/52', - 'BigCodeBench/53', - 'BigCodeBench/54', - 'BigCodeBench/55', - 'BigCodeBench/56', - 'BigCodeBench/57', - 'BigCodeBench/58', - 'BigCodeBench/60', - 'BigCodeBench/61', - 'BigCodeBench/62', - 'BigCodeBench/63', - 'BigCodeBench/64', - 'BigCodeBench/65', - 'BigCodeBench/66', - 'BigCodeBench/67', - 'BigCodeBench/68', - 'BigCodeBench/69', - 'BigCodeBench/70', - 'BigCodeBench/71', - 'BigCodeBench/72', - 'BigCodeBench/73', - 'BigCodeBench/75', - 'BigCodeBench/84', - 'BigCodeBench/85', - 'BigCodeBench/86', - 'BigCodeBench/87', - 'BigCodeBench/88', - 'BigCodeBench/89', - 'BigCodeBench/90', - 'BigCodeBench/91', - 'BigCodeBench/92', - 'BigCodeBench/93', - 'BigCodeBench/94', - 'BigCodeBench/95', - 'BigCodeBench/96', - 'BigCodeBench/97', - 'BigCodeBench/98', - 'BigCodeBench/99', - 'BigCodeBench/100', - 'BigCodeBench/102', - 'BigCodeBench/103', - 'BigCodeBench/104', - 'BigCodeBench/105', - 'BigCodeBench/106', - 'BigCodeBench/107', - 'BigCodeBench/108', - 'BigCodeBench/109', - 'BigCodeBench/110', - 'BigCodeBench/111', - 'BigCodeBench/112', - 'BigCodeBench/113', - 'BigCodeBench/114', - 'BigCodeBench/115', - 'BigCodeBench/116', - 'BigCodeBench/117', - 'BigCodeBench/118', - 'BigCodeBench/119', - 'BigCodeBench/120', - 'BigCodeBench/121', - 'BigCodeBench/122', - 'BigCodeBench/123', - 'BigCodeBench/124', - 'BigCodeBench/125', - 'BigCodeBench/126', - 'BigCodeBench/127', - 'BigCodeBench/128', - 'BigCodeBench/130', - 'BigCodeBench/131', - 'BigCodeBench/132', - 'BigCodeBench/133', - 'BigCodeBench/134', - 'BigCodeBench/135', - 'BigCodeBench/136', - 'BigCodeBench/137', - 'BigCodeBench/138', - 'BigCodeBench/139', - 'BigCodeBench/140', - 'BigCodeBench/141', - 'BigCodeBench/142', - 'BigCodeBench/143', - 'BigCodeBench/148', - 'BigCodeBench/149', - 'BigCodeBench/150', - 'BigCodeBench/151', - 'BigCodeBench/152', - 'BigCodeBench/153', - 'BigCodeBench/155', - 'BigCodeBench/156', - 'BigCodeBench/157', - 'BigCodeBench/159', - 'BigCodeBench/160', - 'BigCodeBench/161', - 'BigCodeBench/162', - 'BigCodeBench/163', - 'BigCodeBench/164', - 'BigCodeBench/165', - 'BigCodeBench/166', - 'BigCodeBench/167', - 'BigCodeBench/168', - 'BigCodeBench/169', - 'BigCodeBench/171', - 'BigCodeBench/172', - 'BigCodeBench/173', - 'BigCodeBench/174', - 'BigCodeBench/175', - 'BigCodeBench/177', - 'BigCodeBench/178', - 'BigCodeBench/179', - 'BigCodeBench/180', - 'BigCodeBench/182', - 'BigCodeBench/184', - 'BigCodeBench/187', - 'BigCodeBench/190', - 'BigCodeBench/191', - 'BigCodeBench/193', - 'BigCodeBench/194', - 'BigCodeBench/196', - 'BigCodeBench/197', - 'BigCodeBench/198', - 'BigCodeBench/199', - 'BigCodeBench/200', - 'BigCodeBench/201', - 'BigCodeBench/202', - 'BigCodeBench/204', - 'BigCodeBench/205', - 'BigCodeBench/206', - 'BigCodeBench/208', - 'BigCodeBench/209', - 'BigCodeBench/210', - 'BigCodeBench/212', - 'BigCodeBench/213', - 'BigCodeBench/214', - 'BigCodeBench/216', - 'BigCodeBench/217', - 'BigCodeBench/218', - 'BigCodeBench/219', - 'BigCodeBench/220', - 'BigCodeBench/221', - 'BigCodeBench/222', - 'BigCodeBench/223', - 'BigCodeBench/224', - 'BigCodeBench/225', - 'BigCodeBench/226', - 'BigCodeBench/227', - 'BigCodeBench/228', - 'BigCodeBench/229', - 'BigCodeBench/230', - 'BigCodeBench/231', - 'BigCodeBench/232', - 'BigCodeBench/233', - 'BigCodeBench/234', - 'BigCodeBench/235', - 'BigCodeBench/236', - 'BigCodeBench/237', - 'BigCodeBench/238', - 'BigCodeBench/239', - 'BigCodeBench/240', - 'BigCodeBench/241', - 'BigCodeBench/242', - 'BigCodeBench/243', - 'BigCodeBench/244', - 'BigCodeBench/245', - 'BigCodeBench/246', - 'BigCodeBench/247', - 'BigCodeBench/248', - 'BigCodeBench/249', - 'BigCodeBench/250', - 'BigCodeBench/251', - 'BigCodeBench/252', - 'BigCodeBench/253', - 'BigCodeBench/254', - 'BigCodeBench/255', - 'BigCodeBench/256', - 'BigCodeBench/257', - 'BigCodeBench/258', - 'BigCodeBench/259', - 'BigCodeBench/260', - 'BigCodeBench/261', - 'BigCodeBench/262', - 'BigCodeBench/263', - 'BigCodeBench/264', - 'BigCodeBench/265', - 'BigCodeBench/266', - 'BigCodeBench/267', - 'BigCodeBench/268', - 'BigCodeBench/269', - 'BigCodeBench/270', - 'BigCodeBench/271', - 'BigCodeBench/275', - 'BigCodeBench/276', - 'BigCodeBench/277', - 'BigCodeBench/278', - 'BigCodeBench/279', - 'BigCodeBench/280', - 'BigCodeBench/281', - 'BigCodeBench/282', - 'BigCodeBench/283', - 'BigCodeBench/286', - 'BigCodeBench/287', - 'BigCodeBench/288', - 'BigCodeBench/289', - 'BigCodeBench/291', - 'BigCodeBench/292', - 'BigCodeBench/293', - 'BigCodeBench/294', - 'BigCodeBench/295', - 'BigCodeBench/296', - 'BigCodeBench/297', - 'BigCodeBench/298', - 'BigCodeBench/299', - 'BigCodeBench/300', - 'BigCodeBench/301', - 'BigCodeBench/302', - 'BigCodeBench/303', - 'BigCodeBench/304', - 'BigCodeBench/305', - 'BigCodeBench/306', - 'BigCodeBench/307', - 'BigCodeBench/308', - 'BigCodeBench/309', - 'BigCodeBench/310', - 'BigCodeBench/311', - 'BigCodeBench/312', - 'BigCodeBench/313', - 'BigCodeBench/316', - 'BigCodeBench/317', - 'BigCodeBench/318', - 'BigCodeBench/319', - 'BigCodeBench/320', - 'BigCodeBench/321', - 'BigCodeBench/322', - 'BigCodeBench/323', - 'BigCodeBench/324', - 'BigCodeBench/325', - 'BigCodeBench/326', - 'BigCodeBench/327', - 'BigCodeBench/328', - 'BigCodeBench/329', - 'BigCodeBench/330', - 'BigCodeBench/331', - 'BigCodeBench/333', - 'BigCodeBench/334', - 'BigCodeBench/335', - 'BigCodeBench/336', - 'BigCodeBench/337', - 'BigCodeBench/338', - 'BigCodeBench/340', - 'BigCodeBench/341', - 'BigCodeBench/342', - 'BigCodeBench/343', - 'BigCodeBench/344', - 'BigCodeBench/345', - 'BigCodeBench/346', - 'BigCodeBench/347', - 'BigCodeBench/348', - 'BigCodeBench/349', - 'BigCodeBench/350', - 'BigCodeBench/351', - 'BigCodeBench/352', - 'BigCodeBench/353', - 'BigCodeBench/354', - 'BigCodeBench/355', - 'BigCodeBench/356', - 'BigCodeBench/357', - 'BigCodeBench/358', - 'BigCodeBench/359', - 'BigCodeBench/360', - 'BigCodeBench/361', - 'BigCodeBench/362', - 'BigCodeBench/363', - 'BigCodeBench/364', - 'BigCodeBench/365', - 'BigCodeBench/366', - 'BigCodeBench/367', - 'BigCodeBench/368', - 'BigCodeBench/369', - 'BigCodeBench/370', - 'BigCodeBench/371', - 'BigCodeBench/372', - 'BigCodeBench/373', - 'BigCodeBench/374', - 'BigCodeBench/375', - 'BigCodeBench/377', - 'BigCodeBench/378', - 'BigCodeBench/379', - 'BigCodeBench/380', - 'BigCodeBench/381', - 'BigCodeBench/382', - 'BigCodeBench/383', - 'BigCodeBench/384', - 'BigCodeBench/385', - 'BigCodeBench/386', - 'BigCodeBench/387', - 'BigCodeBench/388', - 'BigCodeBench/389', - 'BigCodeBench/391', - 'BigCodeBench/392', - 'BigCodeBench/393', - 'BigCodeBench/394', - 'BigCodeBench/395', - 'BigCodeBench/396', - 'BigCodeBench/398', - 'BigCodeBench/399', - 'BigCodeBench/400', - 'BigCodeBench/403', - 'BigCodeBench/404', - 'BigCodeBench/405', - 'BigCodeBench/406', - 'BigCodeBench/407', - 'BigCodeBench/408', - 'BigCodeBench/409', - 'BigCodeBench/410', - 'BigCodeBench/411', - 'BigCodeBench/412', - 'BigCodeBench/414', - 'BigCodeBench/415', - 'BigCodeBench/416', - 'BigCodeBench/417', - 'BigCodeBench/418', - 'BigCodeBench/419', - 'BigCodeBench/420', - 'BigCodeBench/422', - 'BigCodeBench/423', - 'BigCodeBench/424', - 'BigCodeBench/425', - 'BigCodeBench/426', - 'BigCodeBench/427', - 'BigCodeBench/428', - 'BigCodeBench/429', - 'BigCodeBench/430', - 'BigCodeBench/431', - 'BigCodeBench/432', - 'BigCodeBench/433', - 'BigCodeBench/434', - 'BigCodeBench/435', - 'BigCodeBench/436', - 'BigCodeBench/437', - 'BigCodeBench/438', - 'BigCodeBench/439', - 'BigCodeBench/440', - 'BigCodeBench/441', - 'BigCodeBench/442', - 'BigCodeBench/443', - 'BigCodeBench/444', - 'BigCodeBench/445', - 'BigCodeBench/446', - 'BigCodeBench/447', - 'BigCodeBench/448', - 'BigCodeBench/449', - 'BigCodeBench/450', - 'BigCodeBench/451', - 'BigCodeBench/452', - 'BigCodeBench/453', - 'BigCodeBench/454', - 'BigCodeBench/455', - 'BigCodeBench/456', - 'BigCodeBench/457', - 'BigCodeBench/458', - 'BigCodeBench/459', - 'BigCodeBench/460', - 'BigCodeBench/461', - 'BigCodeBench/462', - 'BigCodeBench/463', - 'BigCodeBench/464', - 'BigCodeBench/465', - 'BigCodeBench/466', - 'BigCodeBench/467', - 'BigCodeBench/468', - 'BigCodeBench/469', - 'BigCodeBench/470', - 'BigCodeBench/471', - 'BigCodeBench/472', - 'BigCodeBench/473', - 'BigCodeBench/474', - 'BigCodeBench/475', - 'BigCodeBench/476', - 'BigCodeBench/477', - 'BigCodeBench/478', - 'BigCodeBench/479', - 'BigCodeBench/480', - 'BigCodeBench/481', - 'BigCodeBench/482', - 'BigCodeBench/483', - 'BigCodeBench/484', - 'BigCodeBench/485', - 'BigCodeBench/486', - 'BigCodeBench/487', - 'BigCodeBench/488', - 'BigCodeBench/489', - 'BigCodeBench/491', - 'BigCodeBench/492', - 'BigCodeBench/493', - 'BigCodeBench/494', - 'BigCodeBench/495', - 'BigCodeBench/496', - 'BigCodeBench/497', - 'BigCodeBench/499', - 'BigCodeBench/500', - 'BigCodeBench/501', - 'BigCodeBench/502', - 'BigCodeBench/503', - 'BigCodeBench/504', - 'BigCodeBench/505', - 'BigCodeBench/506', - 'BigCodeBench/507', - 'BigCodeBench/508', - 'BigCodeBench/509', - 'BigCodeBench/510', - 'BigCodeBench/511', - 'BigCodeBench/512', - 'BigCodeBench/513', - 'BigCodeBench/514', - 'BigCodeBench/515', - 'BigCodeBench/516', - 'BigCodeBench/517', - 'BigCodeBench/518', - 'BigCodeBench/519', - 'BigCodeBench/520', - 'BigCodeBench/521', - 'BigCodeBench/522', - 'BigCodeBench/523', - 'BigCodeBench/524', - 'BigCodeBench/525', - 'BigCodeBench/526', - 'BigCodeBench/527', - 'BigCodeBench/528', - 'BigCodeBench/529', - 'BigCodeBench/530', - 'BigCodeBench/531', - 'BigCodeBench/532', - 'BigCodeBench/533', - 'BigCodeBench/534', - 'BigCodeBench/535', - 'BigCodeBench/536', - 'BigCodeBench/537', - 'BigCodeBench/538', - 'BigCodeBench/539', - 'BigCodeBench/540', - 'BigCodeBench/541', - 'BigCodeBench/542', - 'BigCodeBench/543', - 'BigCodeBench/544', - 'BigCodeBench/545', - 'BigCodeBench/546', - 'BigCodeBench/547', - 'BigCodeBench/548', - 'BigCodeBench/549', - 'BigCodeBench/550', - 'BigCodeBench/551', - 'BigCodeBench/552', - 'BigCodeBench/553', - 'BigCodeBench/554', - 'BigCodeBench/555', - 'BigCodeBench/556', - 'BigCodeBench/557', - 'BigCodeBench/558', - 'BigCodeBench/559', - 'BigCodeBench/560', - 'BigCodeBench/561', - 'BigCodeBench/562', - 'BigCodeBench/563', - 'BigCodeBench/564', - 'BigCodeBench/565', - 'BigCodeBench/566', - 'BigCodeBench/567', - 'BigCodeBench/568', - 'BigCodeBench/569', - 'BigCodeBench/570' + '0', + '1', + '2', + '3', + '4', + '5', + '6', + '7', + '8', + '9', + '10', + '11', + '12', + '14', + '15', + '16', + '17', + '18', + '19', + '20', + '21', + '22', + '23', + '24', + '25', + '26', + '27', + '29', + '30', + '31', + '33', + '34', + '35', + '36', + '37', + '38', + '39', + '40', + '41', + '42', + '43', + '44', + '45', + '46', + '47', + '48', + '49', + '50', + '51', + '52', + '53', + '54', + '55', + '56', + '57', + '58', + '60', + '61', + '62', + '63', + '64', + '65', + '66', + '67', + '68', + '69', + '70', + '71', + '72', + '73', + '75', + '84', + '85', + '86', + '87', + '88', + '89', + '90', + '91', + '92', + '93', + '94', + '95', + '96', + '97', + '98', + '99', + '100', + '102', + '103', + '104', + '105', + '106', + '107', + '108', + '109', + '110', + '111', + '112', + '113', + '114', + '115', + '116', + '117', + '118', + '119', + '120', + '121', + '122', + '123', + '124', + '125', + '126', + '127', + '128', + '130', + '131', + '132', + '133', + '134', + '135', + '136', + '137', + '138', + '139', + '140', + '141', + '142', + '143', + '148', + '149', + '150', + '151', + '152', + '153', + '155', + '156', + '157', + '159', + '160', + '161', + '162', + '163', + '164', + '165', + '166', + '167', + '168', + '169', + '171', + '172', + '173', + '174', + '175', + '177', + '178', + '179', + '180', + '182', + '184', + '187', + '190', + '191', + '193', + '194', + '196', + '197', + '198', + '199', + '200', + '201', + '202', + '204', + '205', + '206', + '208', + '209', + '210', + '212', + '213', + '214', + '216', + '217', + '218', + '219', + '220', + '221', + '222', + '223', + '224', + '225', + '226', + '227', + '228', + '229', + '230', + '231', + '232', + '233', + '234', + '235', + '236', + '237', + '238', + '239', + '240', + '241', + '242', + '243', + '244', + '245', + '246', + '247', + '248', + '249', + '250', + '251', + '252', + '253', + '254', + '255', + '256', + '257', + '258', + '259', + '260', + '261', + '262', + '263', + '264', + '265', + '266', + '267', + '268', + '269', + '270', + '271', + '275', + '276', + '277', + '278', + '279', + '280', + '281', + '282', + '283', + '286', + '287', + '288', + '289', + '291', + '292', + '293', + '294', + '295', + '296', + '297', + '298', + '299', + '300', + '301', + '302', + '303', + '304', + '305', + '306', + '307', + '308', + '309', + '310', + '311', + '312', + '313', + '316', + '317', + '318', + '319', + '320', + '321', + '322', + '323', + '324', + '325', + '326', + '327', + '328', + '329', + '330', + '331', + '333', + '334', + '335', + '336', + '337', + '338', + '340', + '341', + '342', + '343', + '344', + '345', + '346', + '347', + '348', + '349', + '350', + '351', + '352', + '353', + '354', + '355', + '356', + '357', + '358', + '359', + '360', + '361', + '362', + '363', + '364', + '365', + '366', + '367', + '368', + '369', + '370', + '371', + '372', + '373', + '374', + '375', + '377', + '378', + '379', + '380', + '381', + '382', + '383', + '384', + '385', + '386', + '387', + '388', + '389', + '391', + '392', + '393', + '394', + '395', + '396', + '398', + '399', + '400', + '403', + '404', + '405', + '406', + '407', + '408', + '409', + '410', + '411', + '412', + '414', + '415', + '416', + '417', + '418', + '419', + '420', + '422', + '423', + '424', + '425', + '426', + '427', + '428', + '429', + '430', + '431', + '432', + '433', + '434', + '435', + '436', + '437', + '438', + '439', + '440', + '441', + '442', + '443', + '444', + '445', + '446', + '447', + '448', + '449', + '450', + '451', + '452', + '453', + '454', + '455', + '456', + '457', + '458', + '459', + '460', + '461', + '462', + '463', + '464', + '465', + '466', + '467', + '468', + '469', + '470', + '471', + '472', + '473', + '474', + '475', + '476', + '477', + '478', + '479', + '480', + '481', + '482', + '483', + '484', + '485', + '486', + '487', + '488', + '489', + '491', + '492', + '493', + '494', + '495', + '496', + '497', + '499', + '500', + '501', + '502', + '503', + '504', + '505', + '506', + '507', + '508', + '509', + '510', + '511', + '512', + '513', + '514', + '515', + '516', + '517', + '518', + '519', + '520', + '521', + '522', + '523', + '524', + '525', + '526', + '527', + '528', + '529', + '530', + '531', + '532', + '533', + '534', + '535', + '536', + '537', + '538', + '539', + '540', + '541', + '542', + '543', + '544', + '545', + '546', + '547', + '548', + '549', + '550', + '551', + '552', + '553', + '554', + '555', + '556', + '557', + '558', + '559', + '560', + '561', + '562', + '563', + '564', + '565', + '566', + '567', + '568', + '569', + '570' ) \ No newline at end of file From b3734b8ccaaca8bf60eaa40ca2bf4959a86f0f6a Mon Sep 17 00:00:00 2001 From: imamnurby Date: Wed, 17 Jul 2024 20:58:21 +0800 Subject: [PATCH 14/14] remove `BigCodeBench/` when getting the instance id --- bigcodebench/data/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigcodebench/data/utils.py b/bigcodebench/data/utils.py index db4c44a..8a2a524 100644 --- a/bigcodebench/data/utils.py +++ b/bigcodebench/data/utils.py @@ -37,7 +37,7 @@ def make_cache(gzip_url, hf_data, cache_path, gh=False, offline=False): # If offline, then parse the json then check the task_id if offline: json_data = [json.loads(line) for line in data.split('\n') if line] - json_data = [item for item in json_data if item.get("task_id") in OFFLINE_IDS] + json_data = [item for item in json_data if item.get("task_id").lstrip("BigCodeBench/") in OFFLINE_IDS] # create CACHE_DIR if not exists if not os.path.exists(CACHE_DIR): @@ -52,7 +52,7 @@ def make_cache(gzip_url, hf_data, cache_path, gh=False, offline=False): f.write(data) else: if offline: - hf_data = hf_data.filter(lambda instance: instance["task_id"] in OFFLINE_IDS) + hf_data = hf_data.filter(lambda instance: instance["task_id"].lstrip("BigCodeBench/") in OFFLINE_IDS) hf_data.to_json(cache_path)