Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit05b916d

Browse files
authored
Merge pull request#4 from codefuse-ai/fc_dev
add funccall evalution features
2 parents338ed89 +f7e2dff commit05b916d

28 files changed

+1611
-0
lines changed

‎scripts/run_fc_example.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
3+
4+
python src/qwen_eval_main.py

‎src/datasets/__init__.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
from .base_datasetimportToolDataset
2+
from .toolfill_datasetimportToolFillDataset
3+
from .toolparser_datasetimportToolParserDataset
4+
from .toolsummary_datasetimportToolSummaryDataset
5+
from .funccall_datasetimportFuncCallDataset
6+
7+
__all__= [
8+
"ToolFillDataset","ToolDataset","ToolParserDataset","ToolSummaryDataset","FuncCallDataset"
9+
]

‎src/datasets/base_dataset.py

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
fromsrc.utils.jsonl_utilsimportread_jsonl_file,save_to_jsonl_file
2+
fromsrc.utils.json_utilsimportread_json_file,save_to_json_file
3+
4+
5+
6+
classToolDataset:
7+
def__init__(self,dataset_name,tool_task,filepath):
8+
self.dataset_name=dataset_name
9+
self.tool_task=tool_task
10+
self.filepath=filepath
11+
self.datas=self.load_data()
12+
13+
defload_data(self, )->list:
14+
ifself.filepath:
15+
returnself.load_data_from_local(self.filepath)
16+
elifself.dataset_nameandself.tool_task:
17+
returnself.load_data_from_hf(self.tool_task)
18+
return []
19+
20+
defload_data_from_local(self,filepath):
21+
''''''
22+
pass
23+
24+
defload_data_from_hf(self,tool_task):
25+
pass
26+
27+
def__iter__(self):
28+
self.current_index=0
29+
returnself
30+
31+
def__next__(self):
32+
ifself.current_index<len(self.datas):
33+
current_item=self.datas[self.current_index]
34+
self.current_index+=1
35+
returncurrent_item
36+
else:
37+
raiseStopIteration
38+
39+
def__len__(self):
40+
returnlen(self.datas)

‎src/datasets/funccall_dataset.py

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
fromsrc.utils.jsonl_utilsimportread_jsonl_file,save_to_jsonl_file
2+
fromsrc.utils.json_utilsimportread_json_file,save_to_json_file
3+
from .base_datasetimportToolDataset
4+
5+
importos
6+
7+
8+
9+
10+
classFuncCallDataset(ToolDataset):
11+
def__init__(self,dataset_name,tool_task,filepath):
12+
self.dataset_name=dataset_name
13+
self.tool_task=tool_task
14+
self.filepath=filepath
15+
self.datas=self.load_data()
16+
17+
defload_data(self, )->list:
18+
ifself.filepath:
19+
returnself.load_data_from_local(self.filepath)
20+
elifself.dataset_nameandself.tool_task:
21+
returnself.load_data_from_hf(self.tool_task)
22+
return []
23+
24+
defload_data_from_local(self,filepath):
25+
def_load_from_file(filename):
26+
if"jsonl"infilename:
27+
returnread_jsonl_file(filename)
28+
elif"json"infilename:
29+
returnread_json_file(filename)
30+
31+
datas= []
32+
ifos.path.isdir(filepath):
33+
forfilenameinos.listdir(filepath):
34+
datas.extend(_load_from_file(os.path.join(filepath,filename)))
35+
else:
36+
datas=_load_from_file(filepath)
37+
38+
returndatas
39+
40+
defload_data_from_hf(self,tool_task):
41+
pass

‎src/datasets/toolfill_dataset.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
fromsrc.utils.jsonl_utilsimportread_jsonl_file,save_to_jsonl_file
2+
fromsrc.utils.json_utilsimportread_json_file,save_to_json_file
3+
from .base_datasetimportToolDataset
4+
5+
6+
7+
8+
classToolFillDataset(ToolDataset):
9+
def__init__(self,dataset_name,tool_task,filepath):
10+
self.dataset_name=dataset_name
11+
self.tool_task=tool_task
12+
self.filepath=filepath
13+
self.datas=self.load_data()
14+
15+
defload_data(self, )->list:
16+
ifself.filepath:
17+
returnself.load_data_from_local(self.filepath)
18+
elifself.dataset_nameandself.tool_task:
19+
returnself.load_data_from_hf(self.tool_task)
20+
return []
21+
22+
defload_data_from_local(self,filepath):
23+
if"jsonl"infilepath:
24+
returnread_jsonl_file(filepath)
25+
elif"json"infilepath:
26+
returnread_json_file(filepath)
27+
return []
28+
29+
defload_data_from_hf(self,tool_task):
30+
pass

‎src/datasets/toolparser_dataset.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
fromsrc.utils.jsonl_utilsimportread_jsonl_file,save_to_jsonl_file
2+
fromsrc.utils.json_utilsimportread_json_file,save_to_json_file
3+
from .base_datasetimportToolDataset
4+
5+
6+
7+
8+
classToolParserDataset(ToolDataset):
9+
def__init__(self,dataset_name,tool_task,filepath):
10+
self.dataset_name=dataset_name
11+
self.tool_task=tool_task
12+
self.filepath=filepath
13+
self.datas=self.load_data()
14+
15+
defload_data(self, )->list:
16+
ifself.filepath:
17+
returnself.load_data_from_local(self.filepath)
18+
elifself.dataset_nameandself.tool_task:
19+
returnself.load_data_from_hf(self.tool_task)
20+
return []
21+
22+
defload_data_from_local(self,filepath):
23+
if"jsonl"infilepath:
24+
returnread_jsonl_file(filepath)
25+
elif"json"infilepath:
26+
returnread_json_file(filepath)
27+
return []
28+
29+
defload_data_from_hf(self,tool_task):
30+
pass

‎src/datasets/toolsummary_dataset.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
fromsrc.utils.jsonl_utilsimportread_jsonl_file,save_to_jsonl_file
2+
fromsrc.utils.json_utilsimportread_json_file,save_to_json_file
3+
from .base_datasetimportToolDataset
4+
5+
6+
classToolSummaryDataset(ToolDataset):
7+
def__init__(self,dataset_name,tool_task,filepath):
8+
self.dataset_name=dataset_name
9+
self.tool_task=tool_task
10+
self.filepath=filepath
11+
self.datas=self.load_data()
12+
13+
defload_data(self, )->list:
14+
ifself.filepath:
15+
returnself.load_data_from_local(self.filepath)
16+
elifself.dataset_nameandself.tool_task:
17+
returnself.load_data_from_hf(self.tool_task)
18+
return []
19+
20+
defload_data_from_local(self,filepath):
21+
if"jsonl"infilepath:
22+
returnread_jsonl_file(filepath)
23+
elif"json"infilepath:
24+
returnread_json_file(filepath)
25+
return []
26+
27+
defload_data_from_hf(self,tool_task):
28+
pass

‎src/evals/__init__.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from .base_evalutionimportToolEvalution
2+
from .toolfill_evalutionimportToolFillEvalution
3+
from .toolparser_evalutionimportToolParserEvalution
4+
from .toolsummary_evalutionimportToolSummaryEvalution
5+
from .func_call_evalutionimportFuncCallEvalution
6+
7+
8+
__all__= [
9+
"ToolEvalution","ToolFillEvalution","ToolParserEvalution","ToolSummaryEvalution","FuncCallEvalution"
10+
]

‎src/evals/base_evalution.py

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
fromsrc.models.base_modelimportToolModel
2+
fromsrc.models.generate_configsimportGenerateConfigs
3+
fromsrc.datasetsimportToolFillDataset
4+
5+
6+
7+
classToolEvalution:
8+
def__init__(
9+
self,
10+
model:ToolModel,
11+
dataset:ToolFillDataset,
12+
base_prompt:str='',
13+
generate_configs:GenerateConfigs=None,
14+
):
15+
self.model=model
16+
self.dataset=dataset
17+
self.base_prompt=base_prompt
18+
self.generate_configs=generate_configs
19+
20+
ifnotisinstance(model,ToolModel):
21+
raiseBaseException(f"must be ToolModel Class! not{model}")
22+
23+
defcalc(self):
24+
'''开始计算结果'''
25+
self.predicts= []
26+
foridx,datainenumerate(self.dataset):
27+
# if idx >= 5: break
28+
prompt=self.base_prompt.format(**data)
29+
answer=data["api_param"]
30+
predict=self.generate(prompt,self.generate_configs)
31+
self.predicts.append({"prompt":prompt,"predict":predict,"answer":answer})
32+
33+
metric=self.eval_metric(self.predicts)
34+
returnmetric
35+
36+
defgenerate(self,prompt,generate_configs):
37+
'''返回结果'''
38+
returnself.model.generate(prompt,generate_configs)
39+
40+
defeval_metric(self,datas):
41+
'''calc custom metric'''
42+
pass
43+

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp