every_eval_ever/ifeval.json
6.0 KB · 211 lines · json Raw
1 {
2 "schema_version": "0.2.2",
3 "evaluation_id": "ifeval/RedHatAI/gemma-4-31B-it-FP8-block/1781646636.638668",
4 "evaluation_timestamp": "1781542156",
5 "retrieved_timestamp": "1781646636.638668",
6 "source_metadata": {
7 "source_name": "lm-evaluation-harness",
8 "source_type": "evaluation_run",
9 "source_organization_name": "RedHatAI",
10 "evaluator_relationship": "third_party"
11 },
12 "eval_library": {
13 "name": "lm_eval",
14 "version": "0.4.13.dev0"
15 },
16 "model_info": {
17 "name": "RedHatAI/gemma-4-31B-it-FP8-block",
18 "id": "RedHatAI/gemma-4-31B-it-FP8-block",
19 "developer": "RedHatAI",
20 "additional_details": {
21 "model_args": "{'model': 'RedHatAI/gemma-4-31B-it-FP8-block', 'max_length': 69632, 'base_url': 'http://127.0.0.1:8000/v1/chat/completions', 'num_concurrent': 32, 'max_retries': 3, 'tokenized_requests': False, 'tokenizer_backend': None, 'timeout': 3600}",
22 "num_seeds_merged": "3"
23 }
24 },
25 "evaluation_results": [
26 {
27 "evaluation_name": "ifeval",
28 "source_data": {
29 "dataset_name": "ifeval",
30 "source_type": "hf_dataset",
31 "hf_repo": "google/IFEval",
32 "hf_split": "train"
33 },
34 "metric_config": {
35 "evaluation_description": "prompt_level_strict_acc",
36 "lower_is_better": false,
37 "score_type": "continuous",
38 "min_score": 0.0,
39 "max_score": 1.0
40 },
41 "score_details": {
42 "score": 0.9125077017868145,
43 "details": {
44 "seed_scores": "[0.9186691312384473, 0.911275415896488, 0.9075785582255084]",
45 "evaluation_timestamps": "[1781542156, 1781542221, 1781542292]",
46 "seed_values": "[1234, 2345, 3456]"
47 },
48 "uncertainty": {
49 "standard_error": {
50 "value": 0.00326032200993786,
51 "method": "across_seeds"
52 },
53 "num_samples": 3
54 }
55 },
56 "generation_config": {
57 "generation_args": {
58 "temperature": 1.0,
59 "top_p": 0.95,
60 "top_k": 64.0,
61 "max_tokens": 32000,
62 "max_attempts": 1
63 },
64 "additional_details": {
65 "until": "[]",
66 "do_sample": "true",
67 "seed": "1234",
68 "num_fewshot": "0"
69 }
70 }
71 },
72 {
73 "evaluation_name": "ifeval",
74 "source_data": {
75 "dataset_name": "ifeval",
76 "source_type": "hf_dataset",
77 "hf_repo": "google/IFEval",
78 "hf_split": "train"
79 },
80 "metric_config": {
81 "evaluation_description": "inst_level_strict_acc",
82 "lower_is_better": false,
83 "score_type": "continuous",
84 "min_score": 0.0,
85 "max_score": 1.0
86 },
87 "score_details": {
88 "score": 0.9400479616306955,
89 "details": {
90 "seed_scores": "[0.9436450839328537, 0.9388489208633094, 0.9376498800959233]",
91 "evaluation_timestamps": "[1781542156, 1781542221, 1781542292]",
92 "seed_values": "[1234, 2345, 3456]"
93 },
94 "uncertainty": {
95 "standard_error": {
96 "value": 0.0018315650259615554,
97 "method": "across_seeds"
98 },
99 "num_samples": 3
100 }
101 },
102 "generation_config": {
103 "generation_args": {
104 "temperature": 1.0,
105 "top_p": 0.95,
106 "top_k": 64.0,
107 "max_tokens": 32000,
108 "max_attempts": 1
109 },
110 "additional_details": {
111 "until": "[]",
112 "do_sample": "true",
113 "seed": "1234",
114 "num_fewshot": "0"
115 }
116 }
117 },
118 {
119 "evaluation_name": "ifeval",
120 "source_data": {
121 "dataset_name": "ifeval",
122 "source_type": "hf_dataset",
123 "hf_repo": "google/IFEval",
124 "hf_split": "train"
125 },
126 "metric_config": {
127 "evaluation_description": "prompt_level_loose_acc",
128 "lower_is_better": false,
129 "score_type": "continuous",
130 "min_score": 0.0,
131 "max_score": 1.0
132 },
133 "score_details": {
134 "score": 0.9328404189772027,
135 "details": {
136 "seed_scores": "[0.9390018484288355, 0.933456561922366, 0.9260628465804066]",
137 "evaluation_timestamps": "[1781542156, 1781542221, 1781542292]",
138 "seed_values": "[1234, 2345, 3456]"
139 },
140 "uncertainty": {
141 "standard_error": {
142 "value": 0.0037478512201467944,
143 "method": "across_seeds"
144 },
145 "num_samples": 3
146 }
147 },
148 "generation_config": {
149 "generation_args": {
150 "temperature": 1.0,
151 "top_p": 0.95,
152 "top_k": 64.0,
153 "max_tokens": 32000,
154 "max_attempts": 1
155 },
156 "additional_details": {
157 "until": "[]",
158 "do_sample": "true",
159 "seed": "1234",
160 "num_fewshot": "0"
161 }
162 }
163 },
164 {
165 "evaluation_name": "ifeval",
166 "source_data": {
167 "dataset_name": "ifeval",
168 "source_type": "hf_dataset",
169 "hf_repo": "google/IFEval",
170 "hf_split": "train"
171 },
172 "metric_config": {
173 "evaluation_description": "inst_level_loose_acc",
174 "lower_is_better": false,
175 "score_type": "continuous",
176 "min_score": 0.0,
177 "max_score": 1.0
178 },
179 "score_details": {
180 "score": 0.9540367705835332,
181 "details": {
182 "seed_scores": "[0.9580335731414868, 0.9544364508393285, 0.9496402877697842]",
183 "evaluation_timestamps": "[1781542156, 1781542221, 1781542292]",
184 "seed_values": "[1234, 2345, 3456]"
185 },
186 "uncertainty": {
187 "standard_error": {
188 "value": 0.0024311600840520617,
189 "method": "across_seeds"
190 },
191 "num_samples": 3
192 }
193 },
194 "generation_config": {
195 "generation_args": {
196 "temperature": 1.0,
197 "top_p": 0.95,
198 "top_k": 64.0,
199 "max_tokens": 32000,
200 "max_attempts": 1
201 },
202 "additional_details": {
203 "until": "[]",
204 "do_sample": "true",
205 "seed": "1234",
206 "num_fewshot": "0"
207 }
208 }
209 }
210 ]
211 }