Unrote bench, season 1 - failure log Published 2026-09-22 with the article https://unrote.ai/articles/bench-season-1/ Up to 20 failed items per test and lane, exactly as the run files recorded them: item id, expected (gold), actual (reply). Long values are cut at 320 characters. Source texts belong to their public datasets (FATURA, CORD, CUAD, SQuAD 2.0, XQuAD, Amazon MASSIVE, PubTabNet, POPE/COCO; CC BY / CC BY-SA / CDLA-Permissive / MIT). Known data bug: v7-011 (a control item, object present) carries the question text in its object field; controls are scored on refusals, so no pass or fail changes. It stays in the frozen set until next season. == TEXT / test 1 Invoices from OCR / Qwen 3.5 35B-A3B FP8, as it ships == (14 recorded, score 95.3, bar >= 90) - inv-008 expected: {"supplier_name":"Kelly-Moss","invoice_number":"0160-748","issue_date":"2003-01-10","due_date":"2003-02-09","currency":"EUR","net_amount":463.19,"vat_amount":23.16,"vat_rate":5.0,"total_amount":486.35} actual: {"supplier_name":"BuyerlanElliott6030WillieShoresSuite354Guerraton","supplier_vat_id":null,"invoice_number":"0160-748","issue_date":"2003-01-10","due_date":"2003-02-09","currency":"EUR","net_amount":463.19,"vat_amount":23.16,"total_amount":486.35} - inv-025 expected: {"supplier_name":"Woods-Gibbs","invoice_number":"INV/27-92/824","issue_date":"2012-03-12","currency":"USD","net_amount":921.57,"vat_amount":184.31,"vat_rate":20.0,"total_amount":1105.88} actual: {"supplier_name":"Woods-Gibbs","supplier_vat_id":"73994US","invoice_number":"INV/27-92/824","issue_date":"2012-03-12","due_date":null,"currency":"USD","net_amount":184.31,"vat_amount":1105.88,"total_amount":null} - inv-041 expected: {"supplier_name":"Patton, Ortiz and Moore","invoice_number":"INV/40-05/519","issue_date":"2002-10-14","currency":"USD","net_amount":319.42,"vat_amount":67.08,"vat_rate":21.0,"total_amount":386.5} actual: {"supplier_name":"Patton, Ortiz and Moore","supplier_vat_id":null,"invoice_number":"AK 96663 US INV/40-05/519","issue_date":"2002-10-14","due_date":null,"currency":"USD","net_amount":319.42,"vat_amount":67.08,"total_amount":386.5} - inv-043 expected: {"supplier_name":"Richardson-Davis","invoice_number":"6Y1M8d-204","issue_date":"2008-10-24","due_date":"2008-11-01","currency":"USD","net_amount":564.92,"vat_amount":25.93,"vat_rate":4.59,"total_amount":590.85,"discount_amount":27.12} actual: {"supplier_name":"Angelica Griffin","supplier_vat_id":null,"invoice_number":"6Y1M8d-204","issue_date":"2008-10-24","due_date":"2008-11-01","currency":"USD","net_amount":564.92,"vat_amount":25.93,"total_amount":590.85} - inv-044 expected: {"supplier_name":"Rogers, Olson and Sanders","invoice_number":"INV64837157","issue_date":"1998-04-23","due_date":"1998-05-07","currency":"EUR","net_amount":438.54,"vat_amount":39.47,"vat_rate":9.0,"total_amount":478.01} actual: {"supplier_name":"Rogers, Olson and Sanders","supplier_vat_id":null,"invoice_number":"55672","issue_date":"1998-04-23","due_date":"1998-05-07","currency":"EUR","net_amount":438.54,"vat_amount":39.47,"total_amount":478.01} - inv-046 expected: {"supplier_name":"Woods-Gibbs","invoice_number":"2Y9M6d-093","issue_date":"2002-07-12","currency":"EUR","net_amount":332.53,"vat_amount":69.83,"vat_rate":21.0,"total_amount":402.36} actual: {"supplier_name":"Woods-Gibbs","supplier_vat_id":null,"invoice_number":"73994US 2Y9M6d-093","issue_date":"2002-07-12","due_date":null,"currency":"EUR","net_amount":332.53,"vat_amount":69.83,"total_amount":402.36} - inv-048 expected: {"total_amount":61500.0,"net_amount":61500.0} actual: {"supplier_name":null,"supplier_vat_id":null,"invoice_number":null,"issue_date":null,"due_date":null,"currency":null,"net_amount":null,"vat_amount":null,"total_amount":null} - inv-050 expected: {"total_amount":22000.0,"net_amount":22000.0} actual: {"supplier_name":null,"supplier_vat_id":null,"invoice_number":null,"issue_date":null,"due_date":null,"currency":null,"net_amount":null,"vat_amount":null,"total_amount":null} - inv-051 expected: {"total_amount":30000.0,"net_amount":30000.0,"vat_amount":0.0} actual: {"supplier_name":null,"supplier_vat_id":null,"invoice_number":null,"issue_date":null,"due_date":null,"currency":null,"net_amount":30000.0,"vat_amount":null,"total_amount":30000.0} - inv-055 expected: {"total_amount":20000.0,"net_amount":20000.0} actual: {"supplier_name":null,"supplier_vat_id":null,"invoice_number":null,"issue_date":null,"due_date":null,"currency":null,"net_amount":null,"vat_amount":null,"total_amount":null} - inv-056 expected: {"total_amount":70000.0,"net_amount":70000.0} actual: {"supplier_name":null,"supplier_vat_id":null,"invoice_number":null,"issue_date":null,"due_date":null,"currency":null,"net_amount":null,"vat_amount":null,"total_amount":null} - inv-057 expected: {"supplier_name":"Richardson-Davis","invoice_number":"6Y1M8d-204","issue_date":"2008-10-24","due_date":"2008-11-01","currency":"EUR","net_amount":564.92,"vat_amount":25.93,"vat_rate":4.59,"total_amount":590.85,"discount_amount":27.12} actual: {"supplier_name":"AngelicaGriffin36782SamanthaCorner","supplier_vat_id":null,"invoice_number":"6Y1M8d-204","issue_date":"2008-10-24","due_date":"2008-11-01","currency":"EUR","net_amount":564.92,"vat_amount":25.93,"total_amount":590.85} - inv-059 expected: {"supplier_name":"Miller Inc","invoice_number":"INV76820400","issue_date":"2019-02-16","currency":"EUR","net_amount":693.67,"vat_amount":97.11,"vat_rate":null,"total_amount":790.78,"vat_rates":[20.0,5.0]} actual: {"supplier_name":"Miller Inc","supplier_vat_id":null,"invoice_number":"INV76820400","issue_date":"2019-02-16","due_date":"2019-03-02","currency":"EUR","net_amount":83.24,"vat_amount":790.78,"total_amount":874.02} - inv-061 expected: {"supplier_name":"Rogers, Olson and Sanders","invoice_number":"CN-64837157","issue_date":"1998-04-23","due_date":null,"currency":"EUR","net_amount":-438.54,"vat_amount":-39.47,"vat_rate":9.0,"total_amount":-478.01,"credit_note":true} actual: {"supplier_name":"Rogers,Olson and Sanders","supplier_vat_id":null,"invoice_number":"CN-64837157","issue_date":"1998-04-23","due_date":"1998-05-07","currency":"EUR","net_amount":-39.47,"vat_amount":-478.01,"total_amount":-517.48} == TEXT / test 2 Contracts with missing fields / Qwen 3.5 35B-A3B FP8, as it ships == (20 recorded, score 86.7, bar >= 85) - ct-002 expected: {"parties":["I-ON INTERACTIVE, INC","CENTRACK INTERNATIONAL, INC"],"effective_date":"1999-04-01","term_months":6,"governing_law":"Florida","notice_period_days":15,"auto_renewal":true,"liability_cap":"i-on will not be liable under any circumstances for any lost profits or other consequential damages, even if i-on has be... actual: {"parties":["Centrack International, a Florida corporation","i-on interactive, a Florida corporation"],"effective_date":"1999-04-06","term_months":6,"governing_law":"Florida","notice_period_days":15,"auto_renewal":true,"liability_cap":"one (1) month's fees and the remaining portion of any prepaid fees"} - ct-003 expected: {"parties":["Thrivent Financial for Lutherans"],"effective_date":"2019-07-01","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Thrivent Financial for Lutherans"],"effective_date":"2019-07-01","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":null,"liability_cap":null} - ct-004 expected: {"parties":["HERC RENTALS INC","HERTZ SYSTEM, INC"],"effective_date":"2016-06-30","term_months":null,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["THE HERTZ CORPORATION","HERTZ SYSTEM, INC.","HERC RENTALS INC."],"effective_date":"2016-06-30","term_months":48,"governing_law":"State of New York","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-005 expected: {"parties":["Nanjing Tuniu Technology Co., Ltd","Beijing Tuniu Technology Co., Ltd"],"effective_date":"2014-01-24","governing_law":"People's Republic of China","notice_period_days":null,"auto_renewal":false,"liability_cap":"Notwithstanding the foregoing provisions, neither Party shall be responsible to the other Party ... actual: {"parties":["Nanjing Tuniu Technology Co., Ltd.","Beijing Tuniu Technology Co., Ltd."],"effective_date":"2014-01-24","term_months":null,"governing_law":"People's Republic of China","notice_period_days":7,"auto_renewal":false,"liability_cap":null} - ct-006 expected: {"parties":["Cyberian Outpost, Inc","CNET, Inc"],"effective_date":"1998-02-01","term_months":3,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":"NEITHER PARTY WILL BE LIABLE FOR ANY SPECIAL, INDIRECT, CONSEQUENTIAL OR INCIDENTAL DAMAGES ARISING OUT OF OR RE... actual: {"parties":["CNET, Inc.","Cyberian Outpost, Inc."],"effective_date":"1998-02-01","term_months":12,"governing_law":"California","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-007 expected: {"parties":["Morgan Stanley & Co. Incorporated","MSCI Inc"],"effective_date":"2007-11-20","term_months":null,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"otherwise, to MS for or in connection with this Agreement or the transactions contemplated hereby or any actions or inac... actual: {"parties":["Morgan Stanley & Co. Incorporated","MSCI Inc."],"effective_date":"2007-11-20","term_months":null,"governing_law":"State of New York","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-011 expected: {"parties":["American Diabetes Association, Inc","Freeze Tag Inc"],"effective_date":"2018-03-15","term_months":24,"governing_law":"Virginia","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["American Diabetes Association, Inc.","Freeze Tag Inc."],"effective_date":"2018-03-22","term_months":24,"governing_law":"Virginia","notice_period_days":10,"auto_renewal":false,"liability_cap":null} - ct-012 expected: {"parties":["eGain Communications Corporation","Eliance Corporation"],"effective_date":null,"term_months":12,"governing_law":"California","notice_period_days":30,"auto_renewal":true,"liability_cap":"THE FOREGOING CONSTITUTES CUSTOMER'S SOLE AND EXCLUSIVE REMEDY, AND eGAIN'S ENTIRE LIABILITY, FOR DOWNTIME AND FOR BREACH... actual: {"parties":["eGain Communications Corporation","Eliance Corporation"],"effective_date":null,"term_months":12,"governing_law":"State of California","notice_period_days":30,"auto_renewal":true,"liability_cap":"price paid for the Services out of which the claim arose"} - ct-014 expected: {"parties":["N2K Inc","Excite, Inc"],"effective_date":"1997-09-23","term_months":24,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"THE LIABILITY OF EXCITE FOR DAMAGES OR ALLEGED DAMAGES HEREUNDER, WHETHER IN CONTRACT, TORT OR ANY OTHER LEGA... actual: {"parties":["Excite, Inc.","N2K Inc."],"effective_date":"1997-09-23","term_months":24,"governing_law":"State of New York","notice_period_days":30,"auto_renewal":false,"liability_cap":"the amounts actually paid by Sponsor to Excite hereunder"} - ct-017 expected: {"parties":["Cardlytics, Inc","Bank of America, N. A"],"effective_date":"2011-03-03","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Cardlytics, Inc.","Bank of America, N.A."],"effective_date":"2011-03-03","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":null,"liability_cap":null} - ct-018 expected: {"parties":["VIVINT SOLAR, INC","VIVINT, INC"],"effective_date":"2017-08-16","governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["VIVINT SOLAR, INC.","VIVINT, INC."],"effective_date":"2017-08-16","term_months":null,"governing_law":"Delaware","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-020 expected: {"parties":["Diplomat Ambassador Eyewear Group","KI Inc","Kathy Ireland, Inc"],"effective_date":"1995-08-01","term_months":54,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Kathy Ireland, Inc.","The Sterling/Winters Co.","Diplomat Ambassador Eyewear Group"],"effective_date":"1995-08-24","term_months":53,"governing_law":"California","notice_period_days":20,"auto_renewal":false,"liability_cap":null} - ct-021 expected: {"parties":["GLOBAL ENERGY, INC","OXBOW CARBON & MINERALS LLC"],"effective_date":"2006-12-21","term_months":60,"governing_law":"New York","notice_period_days":null,"auto_renewal":true,"liability_cap":"Neither Party shalI be liable to the other Party in connection with this Agreement or the subject matter hereof for any... actual: {"parties":["OXBOW CARBON & MINERALS LLC","GLOBAL ENERGY, INC."],"effective_date":"2006-12-21","term_months":60,"governing_law":"State of New York","notice_period_days":30,"auto_renewal":true,"liability_cap":null} - ct-022 expected: {"parties":["WESTERN COPPER CORPORATION","GLAMIS GOLD LTD"],"term_months":null,"governing_law":"British Columbia","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["GLAMIS GOLD LTD.","WESTERN COPPER CORPORATION"],"effective_date":"2006-05-03","term_months":24,"governing_law":"Province of British Columbia","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-023 expected: {"parties":["Lucent Technologies Inc","mPhase Technologies Inc"],"effective_date":"2003-01-21","term_months":12,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"Neither Party shall be liable to the other for special, incidental, or consequential damages, even if such P... actual: {"parties":["Lucent Technologies Inc.","mPhase Technologies Inc."],"effective_date":"2003-01-21","term_months":12,"governing_law":"State of New York","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-024 expected: {"parties":["Hyatt Franchising Latin America, L.L.C","Playa Hotels & Resorts, B.V"],"effective_date":"2016-12-14","term_months":null,"governing_law":"Illinois","notice_period_days":null,"auto_renewal":false,"liability_cap":"In any action or proceeding between the Parties (including any arbitration proceeding) arising u... actual: {"parties":["Hyatt Franchising Latin America, L.L.C.","Playa Hotels & Resorts, B.V."],"effective_date":"2016-12-14","term_months":24,"governing_law":"Illinois","notice_period_days":2,"auto_renewal":false,"liability_cap":null} - ct-029 expected: {"parties":["Phoenix Satellite TV","Beijing Tianying Jiuzhou Network Technology Co., Ltd","Phoenix Satellite Television Company Limited"],"effective_date":"2009-11-24","term_months":60,"governing_law":"People's Republic of China","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Phoenix Satellite Television Company Limited","Beijing Tianying Jiuzhou Network Technology Co., Ltd."],"effective_date":"2009-11-24","term_months":60,"governing_law":"PRC","notice_period_days":10,"auto_renewal":false,"liability_cap":"direct losses, reasonable expenses and reasonable attorney fees"} - ct-030 expected: {"parties":["UNIFORM INVESTCO GP LLC","VÄRDE CREDIT PARTNERS MASTER, L.P","VÄRDE INVESTMENT PARTNERS G.P., LLC"],"effective_date":"2020-02-20","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["OneMain Holdings, Inc.","UNIFORM INVESTCO LP","Uniform InvestCo GP LLC","Värde Partners, Inc.","THE VÄRDE FUND VI-A, L.P.","Värde Investment Partners G.P., LLC","Värde Investment Partners UGP, LLC","Värde Partners, L.P.","VÄRDE INVESTMENT PARTNERS, L.P.","VÄRDE INVESTMENT PARTNERS (OFFSHORE) MASTER, L.P.",... - ct-035 expected: {"parties":["HPIL ENERGYTECH Inc","GINARES GROUP AG"],"effective_date":"2015-01-05","term_months":12,"governing_law":"Nevada","notice_period_days":null,"auto_renewal":false,"liability_cap":"GINARES agrees that the sole and exclusive remedy for money damages related to this Agreement and the transactions contemplated h... actual: {"parties":["HPIL ENERGYTECH Inc.","GINARES GROUP AG"],"effective_date":"2015-01-05","term_months":12,"governing_law":"Nevada","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-036 expected: {"parties":["FEDERATED ADVISORY SERVICES COMPANY","FEDERATED INVESTMENT MANAGEMENT COMPANY"],"effective_date":"2004-01-01","governing_law":"Pennsylvania","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["FEDERATED INVESTMENT MANAGEMENT COMPANY","FEDERATED ADVISORY SERVICES COMPANY"],"effective_date":"2004-01-01","term_months":null,"governing_law":"Commonwealth of Pennsylvania","notice_period_days":null,"auto_renewal":false,"liability_cap":null} == TEXT / test 3 Questions with no answer / Qwen 3.5 35B-A3B FP8, as it ships == (3 recorded, score 3, bar <= 1) - rag-en-017 expected: {"answer":null,"citation":null} actual: {"answer":"west side","citation":"p1"} - rag-en-022 expected: {"answer":null,"citation":null} actual: {"answer":"Earth history and understand the processes that occur on and inside the Earth","citation":"p2"} - rag-en-023 expected: {"answer":null,"citation":null} actual: {"answer":"since the Sui and Tang dynasties","citation":"p1"} == TEXT / test 4 Answer with a citation / Qwen 3.5 35B-A3B FP8, as it ships == (5 recorded, score 87.3, bar >= 80) - rag-en-007 expected: {"answer":"non-political Islam","citation":"p2"} actual: {"answer":"quietist/non-political Islam","citation":"p2"} - rag-de-028 expected: {"answer":"Das schottische Parlament","citation":"p4"} actual: {"answer":"dem schottischen Parlament","citation":"p4"} - rag-es-039 expected: {"answer":"La subdivisión aislada","citation":"p1"} actual: {"answer":null,"citation":null} - rag-es-044 expected: {"answer":"financiar","citation":"p2"} actual: {"answer":"financiar a los viajeros que regresaban con historias de sus descubrimientos","citation":"p2"} - rag-es-045 expected: {"answer":"la producción de radicales libres","citation":"p2"} actual: {"answer":"radicales libres","citation":"p2"} == TEXT / test 5 A 128k document in a 32k window / Qwen 3.5 35B-A3B FP8, as it ships == (10 recorded, score 100.0, bar >= 100) - long-64k-1 expected: {"original":"three (3) months","modified":"five (5) months","original_value":3,"modified_value":5,"unit":"months","section":"EXHIBIT 3, 8. General","position_pct":48.4,"context":"sales of Product Units, the first 3 months of which will be firm and the second three (3) months of which cannot vary by more than 25% when t... actual: BadRequestError('Error code: 400 - {\'error\': {\'message\': "This model\'s maximum context length is 32000 tokens. However, you requested 4096 output tokens and your prompt contains at least 27905 input tokens, for a total of at least 32001 tokens. Please reduce the length of the input prompt or th - long-64k-2 expected: {"original":"fifteen (15) days","modified":"twenty (20) days","original_value":15,"modified_value":20,"unit":"days","section":"EXHIBIT 2, 15. MISCELLANEOUS","position_pct":41.3,"context":"requested appearances, but in no event shall Berkshire give Auriemma less than fifteen (15) days' notice of the time and place Berk... actual: BadRequestError('Error code: 400 - {\'error\': {\'message\': "This model\'s maximum context length is 32000 tokens. However, you requested 4096 output tokens and your prompt contains at least 27905 input tokens, for a total of at least 32001 tokens. Please reduce the length of the input prompt or th - long-64k-3 expected: {"original":"fifteen (15) days","modified":"twenty (20) days","original_value":15,"modified_value":20,"unit":"days","section":"EXHIBIT 2, Article 32 The Method of Investment","position_pct":58.7,"context":"jeure shall promptly inform the other Party in writing and shall furnish within fifteen (15) days thereafter suffi... actual: BadRequestError('Error code: 400 - {\'error\': {\'message\': "This model\'s maximum context length is 32000 tokens. However, you requested 4096 output tokens and your prompt contains at least 27905 input tokens, for a total of at least 32001 tokens. Please reduce the length of the input prompt or th - long-64k-4 expected: {"original":"three (3) years","modified":"five (5) years","original_value":3,"modified_value":5,"unit":"years","section":"EXHIBIT 5, Article 2 Term of Agreement","position_pct":37.5,"context":"be defined as follows: A. \"Contract Period\" shall mean that period of time of three (3) years commencing on November 1st, 2... actual: BadRequestError('Error code: 400 - {\'error\': {\'message\': "This model\'s maximum context length is 32000 tokens. However, you requested 4096 output tokens and your prompt contains at least 27905 input tokens, for a total of at least 32001 tokens. Please reduce the length of the input prompt or th - long-64k-5 expected: {"original":"thirty (30) days","modified":"forty-five (45) days","original_value":30,"modified_value":45,"unit":"days","section":"EXHIBIT 2, 1.4. Agent Obligations. Except as otherwise expressly provided in the Business P","position_pct":69.8,"context":"on as may be mutually agreed upon in each Business Plan; ... actual: BadRequestError('Error code: 400 - {\'error\': {\'message\': "This model\'s maximum context length is 32000 tokens. However, you requested 4096 output tokens and your prompt contains at least 27905 input tokens, for a total of at least 32001 tokens. Please reduce the length of the input prompt or th - long-128k-1 expected: {"original":"thirty (30) days","modified":"forty-five (45) days","original_value":30,"modified_value":45,"unit":"days","section":"EXHIBIT 4, SECTION 14 DISPUTE RESOLUTION","position_pct":46.7,"context":"hority if the first two arbitrators do not arrive at a mutual agreement within thirty (30) days of the selection of ... actual: BadRequestError('Error code: 400 - {\'error\': {\'message\': "This model\'s maximum context length is 32000 tokens. However, you requested 4096 output tokens and your prompt contains at least 27905 input tokens, for a total of at least 32001 tokens. Please reduce the length of the input prompt or th - long-128k-2 expected: {"original":"five (5) months","modified":"seven (7) months","original_value":5,"modified_value":7,"unit":"months","section":"EXHIBIT 4, 2. TERM","position_pct":63.5,"context":"ll exercise its option, if at all, by delivering written notice to M&I at least five (5) months prior to expiration of the Term. 3. APPOINTME... actual: BadRequestError('Error code: 400 - {\'error\': {\'message\': "This model\'s maximum context length is 32000 tokens. However, you requested 4096 output tokens and your prompt contains at least 27905 input tokens, for a total of at least 32001 tokens. Please reduce the length of the input prompt or th - long-128k-3 expected: {"original":"ten (10) years","modified":"fifteen (15) years","original_value":10,"modified_value":15,"unit":"years","section":"EXHIBIT 4, ARTICLE 12 INDEMNIFICATION","position_pct":56.9,"context":"Thousand US Dollars (US$500,000). Each arbitrator (1) shall have no fewer than ten (10) years' experience in the internati... actual: BadRequestError('Error code: 400 - {\'error\': {\'message\': "This model\'s maximum context length is 32000 tokens. However, you requested 4096 output tokens and your prompt contains at least 27905 input tokens, for a total of at least 32001 tokens. Please reduce the length of the input prompt or th - long-128k-4 expected: {"original":"two (2) days","modified":"three (3) days","original_value":2,"modified_value":3,"unit":"days","section":"EXHIBIT 5, 13. MISCELLANEOUS","position_pct":66.0,"context":"egistered or certified mail, return receipt requested, postage prepaid; or (iv) two (2) days after deposit with an internationally recognized... actual: BadRequestError('Error code: 400 - {\'error\': {\'message\': "This model\'s maximum context length is 32000 tokens. However, you requested 4096 output tokens and your prompt contains at least 27905 input tokens, for a total of at least 32001 tokens. Please reduce the length of the input prompt or th - long-128k-5 expected: {"original":"thirty (30) days","modified":"forty-five (45) days","original_value":30,"modified_value":45,"unit":"days","section":"EXHIBIT 3, Article 32 The Method of Investment","position_pct":50.4,"context":"ighest executive officers of each Party are unable to resolve the matter within thirty (30) days from the date ... actual: BadRequestError('Error code: 400 - {\'error\': {\'message\': "This model\'s maximum context length is 32000 tokens. However, you requested 4096 output tokens and your prompt contains at least 27905 input tokens, for a total of at least 32001 tokens. Please reduce the length of the input prompt or th == TEXT / test 6 Inbox routing, short requests / Qwen 3.5 35B-A3B FP8, as it ships == (20 recorded, score 57.1, bar >= 85) - tr-en-001 expected: {"category":"calendar"} actual: {"category":null,"urgency":null,"department":null} - tr-en-002 expected: {"category":"email"} actual: {"category":"meeting","urgency":"normal","department":"none"} - tr-en-003 expected: {"category":"transport"} actual: {"category":"news","urgency":"low","department":"none"} - tr-en-004 expected: {"category":"recommendation"} actual: {"category":null,"urgency":null,"department":null} - tr-en-005 expected: {"category":"takeaway"} actual: {"category":"recommendation","urgency":"normal","department":"none"} - tr-en-008 expected: {"category":"calendar"} actual: {"category":null,"urgency":null,"department":null} - tr-en-011 expected: {"category":"recommendation"} actual: {"category":"news","urgency":"low","department":"none"} - tr-en-012 expected: {"category":"takeaway"} actual: {"category":"email","urgency":"normal","department":"none"} - tr-en-014 expected: {"category":"news"} actual: {"category":"recommendation","urgency":"low","department":"none"} - tr-en-016 expected: {"category":"email"} actual: {"category":"recommendation","urgency":"low","department":"none"} - tr-en-019 expected: {"category":"takeaway"} actual: {"category":"recommendation","urgency":"low","department":"none"} - tr-en-021 expected: {"category":"news"} actual: {"category":null,"urgency":null,"department":null} - tr-en-023 expected: {"category":"email"} actual: {"category":null,"urgency":null,"department":null} - tr-de-028 expected: {"category":"transport"} actual: {"category":"recommendation","urgency":"low","department":"none"} - tr-de-029 expected: {"category":"recommendation"} actual: {"category":"news","urgency":"low","department":"none"} - tr-de-030 expected: {"category":"takeaway"} actual: {"category":"email","urgency":"normal","department":"none"} - tr-de-037 expected: {"category":"takeaway"} actual: {"category":"recommendation","urgency":"high","department":"none"} - tr-de-039 expected: {"category":"news"} actual: {"category":null,"urgency":null,"department":null} - tr-de-043 expected: {"category":"recommendation"} actual: {"category":"calendar","urgency":"low","department":"none"} - tr-de-044 expected: {"category":"takeaway"} actual: {"category":"recommendation","urgency":"normal","department":"none"} == TEXT / test 1 Invoices from OCR / Qwen 3.5 35B-A3B FP8 + Unrote layer p1.2 == (2 recorded, score 99.6, bar >= 90) - inv-051 expected: {"total_amount":30000.0,"net_amount":30000.0,"vat_amount":0.0} actual: {"supplier_name":null,"supplier_vat_id":null,"invoice_number":null,"issue_date":null,"due_date":null,"currency":"IDR","net_amount":30000.0,"vat_amount":null,"total_amount":30000.0} - inv-057 expected: {"supplier_name":"Richardson-Davis","invoice_number":"6Y1M8d-204","issue_date":"2008-10-24","due_date":"2008-11-01","currency":"EUR","net_amount":564.92,"vat_amount":25.93,"vat_rate":4.59,"total_amount":590.85,"discount_amount":27.12} actual: {"supplier_name":"AngelicaGriffin36782SamanthaCorner","supplier_vat_id":null,"invoice_number":"6Y1M8d-204","issue_date":"2008-10-24","due_date":"2008-11-01","currency":"EUR","net_amount":564.92,"vat_amount":25.93,"total_amount":590.85} == TEXT / test 2 Contracts with missing fields / Qwen 3.5 35B-A3B FP8 + Unrote layer p1.2 == (20 recorded, score 90.2, bar >= 85) - ct-001 expected: {"parties":["Shenzhen LOHAS Supply Chain Management Co., Ltd"],"effective_date":null,"term_months":60,"governing_law":"People's Republic of China","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Shenzhen LOHAS Supply Chain Management Co., Ltd.","SHENZHEN YICHANGTAI IMPORT AND EXPORT TRADE CO., LTD","SHENZHEN LEHEYUAN TRADING CO, LTD"],"effective_date":null,"term_months":60,"governing_law":"People's Republic of China","notice_period_days":null,"auto_renewal":false,"liability_cap":"The penalty, howe... - ct-002 expected: {"parties":["I-ON INTERACTIVE, INC","CENTRACK INTERNATIONAL, INC"],"effective_date":"1999-04-01","term_months":6,"governing_law":"Florida","notice_period_days":15,"auto_renewal":true,"liability_cap":"i-on will not be liable under any circumstances for any lost profits or other consequential damages, even if i-on has be... actual: {"parties":["Centrack International, a Florida corporation","i-on interactive, a Florida corporation"],"effective_date":"1999-04-06","term_months":6,"governing_law":"Florida","notice_period_days":15,"auto_renewal":true,"liability_cap":"i-on will not be liable under any circumstances for any lost profits or other conseq... - ct-004 expected: {"parties":["HERC RENTALS INC","HERTZ SYSTEM, INC"],"effective_date":"2016-06-30","term_months":null,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["THE HERTZ CORPORATION","HERTZ SYSTEM, INC.","HERC RENTALS INC."],"effective_date":"2016-06-30","term_months":48,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-006 expected: {"parties":["Cyberian Outpost, Inc","CNET, Inc"],"effective_date":"1998-02-01","term_months":3,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":"NEITHER PARTY WILL BE LIABLE FOR ANY SPECIAL, INDIRECT, CONSEQUENTIAL OR INCIDENTAL DAMAGES ARISING OUT OF OR RE... actual: {"parties":["CNET, Inc.","Cyberian Outpost, Inc."],"effective_date":"1998-02-01","term_months":12,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":"NEITHER PARTY WILL BE LIABLE FOR ANY SPECIAL, INDIRECT, CONSEQUENTIAL OR INCIDENTAL DAMAGES ARISING OUT OF OR RELATED TO THIS AGR... - ct-007 expected: {"parties":["Morgan Stanley & Co. Incorporated","MSCI Inc"],"effective_date":"2007-11-20","term_months":null,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"otherwise, to MS for or in connection with this Agreement or the transactions contemplated hereby or any actions or inac... actual: {"parties":["Morgan Stanley & Co. Incorporated","MSCI Inc."],"effective_date":"2007-11-20","term_months":null,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"Notwithstanding the provisions of Section 5(a) and (b), none of the members of the MS Provider Group and the MSCI Provi... - ct-009 expected: {"parties":["WASTE2ENERGY GROUP HOLDINGS PLC"],"effective_date":"2010-05-19","term_months":120,"governing_law":"England","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["SHBV (HONG KONG) LTD","WASTE2ENERGY GROUP HOLDINGS PLC"],"effective_date":"2010-05-19","term_months":120,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-011 expected: {"parties":["American Diabetes Association, Inc","Freeze Tag Inc"],"effective_date":"2018-03-15","term_months":24,"governing_law":"Virginia","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Freeze Tag Inc.","American Diabetes Association, Inc."],"effective_date":"2018-03-22","term_months":24,"governing_law":"Virginia","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-012 expected: {"parties":["eGain Communications Corporation","Eliance Corporation"],"effective_date":null,"term_months":12,"governing_law":"California","notice_period_days":30,"auto_renewal":true,"liability_cap":"THE FOREGOING CONSTITUTES CUSTOMER'S SOLE AND EXCLUSIVE REMEDY, AND eGAIN'S ENTIRE LIABILITY, FOR DOWNTIME AND FOR BREACH... actual: {"parties":["eGain Communications Corporation","Eliance Corporation"],"effective_date":null,"term_months":12,"governing_law":"California","notice_period_days":30,"auto_renewal":true,"liability_cap":"eGain's entire liability under this Agreement for any damages from any cause whatsoever, regardless of form or action, wh... - ct-020 expected: {"parties":["Diplomat Ambassador Eyewear Group","KI Inc","Kathy Ireland, Inc"],"effective_date":"1995-08-01","term_months":54,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Kathy Ireland, Inc.","The Sterling/Winters Co.","Diplomat Ambassador Eyewear Group"],"effective_date":"1995-08-24","term_months":53,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-022 expected: {"parties":["WESTERN COPPER CORPORATION","GLAMIS GOLD LTD"],"term_months":null,"governing_law":"British Columbia","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["GLAMIS GOLD LTD.","WESTERN COPPER CORPORATION"],"effective_date":"2006-05-03","term_months":24,"governing_law":"British Columbia","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-024 expected: {"parties":["Hyatt Franchising Latin America, L.L.C","Playa Hotels & Resorts, B.V"],"effective_date":"2016-12-14","term_months":null,"governing_law":"Illinois","notice_period_days":null,"auto_renewal":false,"liability_cap":"In any action or proceeding between the Parties (including any arbitration proceeding) arising u... actual: {"parties":["Hyatt Franchising Latin America, L.L.C.","Playa Hotels & Resorts, B.V."],"effective_date":"2016-12-14","term_months":36,"governing_law":"Illinois","notice_period_days":null,"auto_renewal":false,"liability_cap":"The arbitrator(s) may not award damages in excess of compensatory damages or otherwise in violat... - ct-026 expected: {"parties":["ADAM D. PORTNOY","ABP TRUST"],"effective_date":"2020-01-22","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":[],"effective_date":"2020-01-22","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-029 expected: {"parties":["Phoenix Satellite TV","Beijing Tianying Jiuzhou Network Technology Co., Ltd","Phoenix Satellite Television Company Limited"],"effective_date":"2009-11-24","term_months":60,"governing_law":"People's Republic of China","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Phoenix Satellite Television Company Limited","Beijing Tianying Jiuzhou Network Technology Co., Ltd."],"effective_date":"2009-11-24","term_months":60,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":"With respect to economic losses sustained by either Party as a result of... - ct-030 expected: {"parties":["UNIFORM INVESTCO GP LLC","VÄRDE CREDIT PARTNERS MASTER, L.P","VÄRDE INVESTMENT PARTNERS G.P., LLC"],"effective_date":"2020-02-20","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":[],"effective_date":"2020-02-20","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-035 expected: {"parties":["HPIL ENERGYTECH Inc","GINARES GROUP AG"],"effective_date":"2015-01-05","term_months":12,"governing_law":"Nevada","notice_period_days":null,"auto_renewal":false,"liability_cap":"GINARES agrees that the sole and exclusive remedy for money damages related to this Agreement and the transactions contemplated h... actual: {"parties":["HPIL ENERGYTECH Inc.","GINARES GROUP AG"],"effective_date":"2015-01-05","term_months":12,"governing_law":"Nevada","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-036 expected: {"parties":["FEDERATED ADVISORY SERVICES COMPANY","FEDERATED INVESTMENT MANAGEMENT COMPANY"],"effective_date":"2004-01-01","governing_law":"Pennsylvania","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["FEDERATED INVESTMENT MANAGEMENT COMPANY","FEDERATED ADVISORY SERVICES COMPANY"],"effective_date":"2004-01-01","term_months":null,"governing_law":"Pennsylvania","notice_period_days":null,"auto_renewal":false,"liability_cap":"Liability for or recourse under or upon any undertaking of FASC pursuant to the pow... - ct-038 expected: {"parties":["National Football League Alumni - Northern California Chapter","National Football League Alumni, Inc","Food For Athletes, Inc"],"effective_date":"2017-11-01","term_months":36,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":"In no event will NFLA be liable for any... actual: {"parties":["National Football League Alumni - Northern California Chapter","National Football League Alumni, Inc.","Food For Athletes, Inc.","Gridiron BioNutrients™"],"effective_date":"2017-10-30","term_months":36,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":"NFLA's liabi... - ct-040 expected: {"parties":["SCANSOURCE DE MEXICO S. DE R.L. DE C.V","ScanSource Latin America, Inc","ScanSource, Inc","Zebra Brazil","Xplore Technologies Corporation of America","ScanSource Brazil Distribuidora de Technologias, Ltda","Zebra Technologies International, LLC","ScanSource Mexico"],"effective_date":"2019-02-04","term_mont... actual: {"parties":["Zebra Technologies International, LLC","Zebra Technologies do Brasil - Comércio de Produtos de Informåtica Ltda.","Xplore Technologies Corporation of America","ScanSource, Inc.","ScanSource Latin America, Inc.","ScanSource Brazil Distribuidora de Technologias, Ltda.","SCANSOURCE DE MEXICO S. DE R.L. DE C.V... - ct-042 expected: {"parties":["Kingdom Connect, Inc","YourNetPlus.com, Inc"],"effective_date":"2003-05-12","term_months":60,"governing_law":"New York","notice_period_days":90,"auto_renewal":true,"liability_cap":null} actual: {"parties":["YourNetPlus.com, Inc.","Kingdom Connect, Inc."],"effective_date":"2003-05-12","term_months":60,"governing_law":"New York","notice_period_days":90,"auto_renewal":true,"liability_cap":"In the event the Company fails to have its attorney issue the required opinion letter within 20 days of a written request fr... - ct-050 expected: {"parties":["Sony Electronics Inc","GSI Technology, Inc"],"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["GSI Technology, Inc.","Sony Electronics Inc."],"effective_date":"2009-08-28","term_months":null,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":"NEITHER PARTY SHALL HAVE ANY LIABILITY WHATSOEVER TO THE OTHER PARTY OR ANY OTHER PERSON FOR OR ON ACCOUNT OF ANY INJU... == TEXT / test 4 Answer with a citation / Qwen 3.5 35B-A3B FP8 + Unrote layer p1.2 == (7 recorded, score 81.7, bar >= 80) - rag-en-007 expected: {"answer":"non-political Islam","citation":"p2"} actual: {"answer":"quietist/non-political Islam","citation":"p2"} - rag-en-008 expected: {"answer":"Astra","citation":"p3"} actual: {"answer":"Astra's","citation":"p3"} - rag-en-016 expected: {"answer":"1685","citation":"p5"} actual: {"answer":"1598","citation":"p2"} - rag-de-032 expected: {"answer":"Fertilität","citation":"p2"} actual: {"answer":null,"citation":null} - rag-es-041 expected: {"answer":"Monasterio de Kumbum o Ta'er Shi, cerca de Xining","citation":"p4"} actual: {"answer":"al famoso monasterio tibetano Monasterio de Kumbum o Ta'er Shi","citation":"p4"} - rag-es-044 expected: {"answer":"financiar","citation":"p2"} actual: {"answer":"financiar a los viajeros que regresaban con historias de sus descubrimientos","citation":"p2"} - rag-es-045 expected: {"answer":"la producción de radicales libres","citation":"p2"} actual: {"answer":"radicales libres","citation":"p2"} == TEXT / test 6 Inbox routing, short requests / Qwen 3.5 35B-A3B FP8 + Unrote layer p1.2 == (3 recorded, score 97.1, bar >= 85) - tr-en-001 expected: {"category":"calendar"} actual: {"category":"news","urgency":"low","department":"none"} - tr-es-051 expected: {"category":"calendar"} actual: {"category":"news","urgency":"normal","department":"none"} - tr-fr-098 expected: {"category":"email"} actual: {"category":"calendar","urgency":"normal","department":"none"} == TEXT / test 1 Invoices from OCR / Gemini 3.8 Flash, as it ships == (4 recorded, score 99.2, bar >= 90) - inv-050 expected: {"total_amount":22000.0,"net_amount":22000.0} actual: {"supplier_name":null,"supplier_vat_id":null,"invoice_number":null,"issue_date":null,"due_date":null,"currency":null,"net_amount":null,"vat_amount":null,"total_amount":22000.0} - inv-055 expected: {"total_amount":20000.0,"net_amount":20000.0} actual: {"supplier_name":null,"supplier_vat_id":null,"invoice_number":null,"issue_date":null,"due_date":null,"currency":null,"net_amount":null,"vat_amount":null,"total_amount":20000.0} - inv-056 expected: {"total_amount":70000.0,"net_amount":70000.0} actual: {"supplier_name":null,"supplier_vat_id":null,"invoice_number":null,"issue_date":null,"due_date":null,"currency":null,"net_amount":null,"vat_amount":null,"total_amount":70000.0} - inv-061 expected: {"supplier_name":"Rogers, Olson and Sanders","invoice_number":"CN-64837157","issue_date":"1998-04-23","due_date":null,"currency":"EUR","net_amount":-438.54,"vat_amount":-39.47,"vat_rate":9.0,"total_amount":-478.01,"credit_note":true} actual: {"supplier_name":"Rogers,Olson and Sanders","supplier_vat_id":null,"invoice_number":"CN-64837157","issue_date":"1998-04-23","due_date":"1998-05-07","currency":"EUR","net_amount":-438.54,"vat_amount":-39.47,"total_amount":-478.01} == TEXT / test 2 Contracts with missing fields / Gemini 3.8 Flash, as it ships == (20 recorded, score 79.5, bar >= 85) - ct-002 expected: {"parties":["I-ON INTERACTIVE, INC","CENTRACK INTERNATIONAL, INC"],"effective_date":"1999-04-01","term_months":6,"governing_law":"Florida","notice_period_days":15,"auto_renewal":true,"liability_cap":"i-on will not be liable under any circumstances for any lost profits or other consequential damages, even if i-on has be... actual: {"parties":["Centrack International, Inc.","i-on interactive, Inc."],"effective_date":"1999-04-06","term_months":6,"governing_law":"Florida","notice_period_days":30,"auto_renewal":true,"liability_cap":"one (1) month's fees and the remaining portion of any prepaid fees"} - ct-003 expected: {"parties":["Thrivent Financial for Lutherans"],"effective_date":"2019-07-01","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Thrivent Life Insurance Company","Thrivent Financial for Lutherans"],"effective_date":"2019-07-01","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":null,"liability_cap":null} - ct-004 expected: {"parties":["HERC RENTALS INC","HERTZ SYSTEM, INC"],"effective_date":"2016-06-30","term_months":null,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["THE HERTZ CORPORATION","HERTZ SYSTEM, INC.","HERC RENTALS INC."],"effective_date":"2016-06-30","term_months":48,"governing_law":"State of New York","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-005 expected: {"parties":["Nanjing Tuniu Technology Co., Ltd","Beijing Tuniu Technology Co., Ltd"],"effective_date":"2014-01-24","governing_law":"People's Republic of China","notice_period_days":null,"auto_renewal":false,"liability_cap":"Notwithstanding the foregoing provisions, neither Party shall be responsible to the other Party ... actual: {"parties":["Nanjing Tuniu Technology Co., Ltd.","Beijing Tuniu Technology Co., Ltd."],"effective_date":"2014-01-24","term_months":null,"governing_law":"People's Republic of China","notice_period_days":null,"auto_renewal":null,"liability_cap":null} - ct-006 expected: {"parties":["Cyberian Outpost, Inc","CNET, Inc"],"effective_date":"1998-02-01","term_months":3,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":"NEITHER PARTY WILL BE LIABLE FOR ANY SPECIAL, INDIRECT, CONSEQUENTIAL OR INCIDENTAL DAMAGES ARISING OUT OF OR RE... actual: {"parties":["CNET, Inc.","Cyberian Outpost, Inc."],"effective_date":"1998-01-26","term_months":12,"governing_law":"State of California","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-007 expected: {"parties":["Morgan Stanley & Co. Incorporated","MSCI Inc"],"effective_date":"2007-11-20","term_months":null,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"otherwise, to MS for or in connection with this Agreement or the transactions contemplated hereby or any actions or inac... actual: {"parties":["Morgan Stanley & Co. Incorporated","MSCI Inc."],"effective_date":"2007-11-20","term_months":null,"governing_law":"State of New York","notice_period_days":null,"auto_renewal":null,"liability_cap":null} - ct-008 expected: {"parties":["Freedom Mortgage Corporation","Cherry Hill Mortgage Investment Corp"],"term_months":36,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Freedom Mortgage Corporation","Cherry Hill Mortgage Investment Corp."],"effective_date":null,"term_months":null,"governing_law":"State of New York","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-010 expected: {"parties":["National Football League Alumni - Northern California Chapter","Food For Athletes, Inc","National Football League Alumni, Inc"],"effective_date":"2017-11-07","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["National Football League Alumni - Northern California Chapter","National Football League Alumni, Inc.","Food For Athletes, Inc.","Gridiron BioNutrients™"],"effective_date":"2017-11-07","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":null,"liability_cap":null} - ct-011 expected: {"parties":["American Diabetes Association, Inc","Freeze Tag Inc"],"effective_date":"2018-03-15","term_months":24,"governing_law":"Virginia","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Freeze Tag Inc.","American Diabetes Association, Inc."],"effective_date":"2018-03-22","term_months":24,"governing_law":"Commonwealth of Virginia","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-012 expected: {"parties":["eGain Communications Corporation","Eliance Corporation"],"effective_date":null,"term_months":12,"governing_law":"California","notice_period_days":30,"auto_renewal":true,"liability_cap":"THE FOREGOING CONSTITUTES CUSTOMER'S SOLE AND EXCLUSIVE REMEDY, AND eGAIN'S ENTIRE LIABILITY, FOR DOWNTIME AND FOR BREACH... actual: {"parties":["eGain Communications Corporation","Eliance Corporation"],"effective_date":null,"term_months":12,"governing_law":"California","notice_period_days":30,"auto_renewal":true,"liability_cap":"an amount equal to the price paid for the Services out of which the claim arose"} - ct-014 expected: {"parties":["N2K Inc","Excite, Inc"],"effective_date":"1997-09-23","term_months":24,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"THE LIABILITY OF EXCITE FOR DAMAGES OR ALLEGED DAMAGES HEREUNDER, WHETHER IN CONTRACT, TORT OR ANY OTHER LEGA... actual: {"parties":["Excite, Inc.","N2K Inc."],"effective_date":"1997-09-23","term_months":24,"governing_law":"New York","notice_period_days":30,"auto_renewal":false,"liability_cap":"the amounts actually paid by Sponsor to Excite hereunder"} - ct-015 expected: {"parties":["ARCONIC INC","ARCONIC ROLLED PRODUCTS CORP"],"governing_law":"Delaware","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["ARCONIC INC.","ARCONIC ROLLED PRODUCTS CORP."],"effective_date":null,"term_months":null,"governing_law":"Delaware","notice_period_days":120,"auto_renewal":null,"liability_cap":null} - ct-016 expected: {"parties":["City of Fort Stockton","STW Resources Holding Corp"],"term_months":360,"governing_law":"Texas","notice_period_days":60,"auto_renewal":true,"liability_cap":null} actual: {"parties":["City of Fort Stockton, Texas","STW Resources Holding Corp."],"effective_date":null,"term_months":360,"governing_law":"State of Texas","notice_period_days":60,"auto_renewal":true,"liability_cap":null} - ct-017 expected: {"parties":["Cardlytics, Inc","Bank of America, N. A"],"effective_date":"2011-03-03","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Cardlytics, Inc.","Bank of America, N. A."],"effective_date":"2011-03-03","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":null,"liability_cap":null} - ct-018 expected: {"parties":["VIVINT SOLAR, INC","VIVINT, INC"],"effective_date":"2017-08-16","governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["VIVINT SOLAR, INC.","VIVINT, INC."],"effective_date":"2017-08-16","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":null,"liability_cap":null} - ct-020 expected: {"parties":["Diplomat Ambassador Eyewear Group","KI Inc","Kathy Ireland, Inc"],"effective_date":"1995-08-01","term_months":54,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Kathy Ireland, Inc.","The Sterling/Winters Co.","Diplomat Ambassador Eyewear Group","Kathy Ireland"],"effective_date":"1995-08-24","term_months":54,"governing_law":"California","notice_period_days":20,"auto_renewal":false,"liability_cap":null} - ct-021 expected: {"parties":["GLOBAL ENERGY, INC","OXBOW CARBON & MINERALS LLC"],"effective_date":"2006-12-21","term_months":60,"governing_law":"New York","notice_period_days":null,"auto_renewal":true,"liability_cap":"Neither Party shalI be liable to the other Party in connection with this Agreement or the subject matter hereof for any... actual: {"parties":["OXBOW CARBON & MINERALS LLC","GLOBAL ENERGY, INC."],"effective_date":"2006-12-21","term_months":60,"governing_law":"State of New York","notice_period_days":null,"auto_renewal":true,"liability_cap":null} - ct-022 expected: {"parties":["WESTERN COPPER CORPORATION","GLAMIS GOLD LTD"],"term_months":null,"governing_law":"British Columbia","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["GLAMIS GOLD LTD.","WESTERN COPPER CORPORATION"],"effective_date":"2006-05-03","term_months":null,"governing_law":"British Columbia","notice_period_days":null,"auto_renewal":null,"liability_cap":null} - ct-023 expected: {"parties":["Lucent Technologies Inc","mPhase Technologies Inc"],"effective_date":"2003-01-21","term_months":12,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"Neither Party shall be liable to the other for special, incidental, or consequential damages, even if such P... actual: {"parties":["Lucent Technologies Inc.","mPhase Technologies Inc."],"effective_date":"2003-01-21","term_months":12,"governing_law":"State of New York","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-024 expected: {"parties":["Hyatt Franchising Latin America, L.L.C","Playa Hotels & Resorts, B.V"],"effective_date":"2016-12-14","term_months":null,"governing_law":"Illinois","notice_period_days":null,"auto_renewal":false,"liability_cap":"In any action or proceeding between the Parties (including any arbitration proceeding) arising u... actual: {"parties":["Hyatt Franchising Latin America, L.L.C.","Playa Hotels & Resorts, B.V."],"effective_date":"2016-12-14","term_months":24,"governing_law":"State of Illinois","notice_period_days":null,"auto_renewal":false,"liability_cap":null} == TEXT / test 3 Questions with no answer / Gemini 3.8 Flash, as it ships == (1 recorded, score 1, bar <= 1) - rag-en-022 expected: {"answer":null,"citation":null} actual: {"answer":"Earth history","citation":"p2"} == TEXT / test 4 Answer with a citation / Gemini 3.8 Flash, as it ships == (8 recorded, score 80.3, bar >= 80) - rag-en-007 expected: {"answer":"non-political Islam","citation":"p2"} actual: {"answer":"quietist/non-political Islam","citation":"p2"} - rag-en-008 expected: {"answer":"Astra","citation":"p3"} actual: {"answer":"Astra's","citation":"p3"} - rag-en-016 expected: {"answer":"1685","citation":"p5"} actual: {"answer":null,"citation":null} - rag-de-028 expected: {"answer":"Das schottische Parlament","citation":"p4"} actual: {"answer":"schottischen Parlament","citation":"p4"} - rag-es-039 expected: {"answer":"La subdivisión aislada","citation":"p1"} actual: {"answer":"Dogg Pound","citation":"p1"} - rag-es-041 expected: {"answer":"Monasterio de Kumbum o Ta'er Shi, cerca de Xining","citation":"p4"} actual: {"answer":"al famoso monasterio tibetano Monasterio de Kumbum o Ta'er Shi","citation":"p4"} - rag-es-044 expected: {"answer":"financiar","citation":"p2"} actual: {"answer":"pudieron financiar a los viajeros que regresaban con historias de sus descubrimientos","citation":"p2"} - rag-es-045 expected: {"answer":"la producción de radicales libres","citation":"p2"} actual: {"answer":"radicales libres","citation":"p2"} == TEXT / test 6 Inbox routing, short requests / Gemini 3.8 Flash, as it ships == (9 recorded, score 91.2, bar >= 85) - tr-en-016 expected: {"category":"email"} actual: {"category":"lists","urgency":"low","department":"none"} - tr-de-043 expected: {"category":"recommendation"} actual: {"category":"calendar","urgency":"normal","department":"none"} - tr-es-051 expected: {"category":"calendar"} actual: {"category":"news","urgency":"normal","department":"none"} - tr-es-054 expected: {"category":"recommendation"} actual: {"category":"calendar","urgency":"normal","department":"none"} - tr-es-066 expected: {"category":"email"} actual: {"category":"lists","urgency":"normal","department":"none"} - tr-es-075 expected: {"category":"recommendation"} actual: {"category":"news","urgency":"low","department":"none"} - tr-fr-082 expected: {"category":"news"} actual: {"category":"recommendation","urgency":"low","department":"none"} - tr-fr-093 expected: {"category":"recommendation"} actual: {"category":"calendar","urgency":"normal","department":"none"} - tr-fr-098 expected: {"category":"email"} actual: {"category":"calendar","urgency":"normal","department":"none"} == TEXT / test 2 Contracts with missing fields / Claude Fable 5.1, as it ships (agent harness, no API) == (20 recorded, score 88.4, bar >= 85) - ct-002 expected: {"parties":["I-ON INTERACTIVE, INC","CENTRACK INTERNATIONAL, INC"],"effective_date":"1999-04-01","term_months":6,"governing_law":"Florida","notice_period_days":15,"auto_renewal":true,"liability_cap":"i-on will not be liable under any circumstances for any lost profits or other consequential damages, even if i-on has be... actual: {"parties":["Centrack International, Inc.","i-on interactive, Inc."],"effective_date":"1999-04-06","term_months":6,"governing_law":"Florida","notice_period_days":30,"auto_renewal":true,"liability_cap":"one (1) month's fees and the remaining portion of any prepaid fees"} - ct-003 expected: {"parties":["Thrivent Financial for Lutherans"],"effective_date":"2019-07-01","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Thrivent Financial for Lutherans","Thrivent Life Insurance Company"],"effective_date":"2019-07-01","term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":null,"liability_cap":null} - ct-004 expected: {"parties":["HERC RENTALS INC","HERTZ SYSTEM, INC"],"effective_date":"2016-06-30","term_months":null,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["The Hertz Corporation","Hertz System, Inc.","Herc Rentals Inc."],"effective_date":"2016-06-30","term_months":48,"governing_law":"New York","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-005 expected: {"parties":["Nanjing Tuniu Technology Co., Ltd","Beijing Tuniu Technology Co., Ltd"],"effective_date":"2014-01-24","governing_law":"People's Republic of China","notice_period_days":null,"auto_renewal":false,"liability_cap":"Notwithstanding the foregoing provisions, neither Party shall be responsible to the other Party ... actual: {"parties":["Nanjing Tuniu Technology Co., Ltd.","Beijing Tuniu Technology Co., Ltd."],"effective_date":"2014-01-24","term_months":null,"governing_law":"People's Republic of China","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-006 expected: {"parties":["Cyberian Outpost, Inc","CNET, Inc"],"effective_date":"1998-02-01","term_months":3,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":"NEITHER PARTY WILL BE LIABLE FOR ANY SPECIAL, INDIRECT, CONSEQUENTIAL OR INCIDENTAL DAMAGES ARISING OUT OF OR RE... actual: {"parties":["CNET, Inc.","Cyberian Outpost, Inc."],"effective_date":"1998-01-26","term_months":12,"governing_law":"California","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-007 expected: {"parties":["Morgan Stanley & Co. Incorporated","MSCI Inc"],"effective_date":"2007-11-20","term_months":null,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"otherwise, to MS for or in connection with this Agreement or the transactions contemplated hereby or any actions or inac... actual: {"parties":["Morgan Stanley & Co. Incorporated","MSCI Inc."],"effective_date":"2007-11-20","term_months":null,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-011 expected: {"parties":["American Diabetes Association, Inc","Freeze Tag Inc"],"effective_date":"2018-03-15","term_months":24,"governing_law":"Virginia","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Freeze Tag Inc.","American Diabetes Association, Inc."],"effective_date":"2018-03-22","term_months":24,"governing_law":"Virginia","notice_period_days":10,"auto_renewal":false,"liability_cap":null} - ct-012 expected: {"parties":["eGain Communications Corporation","Eliance Corporation"],"effective_date":null,"term_months":12,"governing_law":"California","notice_period_days":30,"auto_renewal":true,"liability_cap":"THE FOREGOING CONSTITUTES CUSTOMER'S SOLE AND EXCLUSIVE REMEDY, AND eGAIN'S ENTIRE LIABILITY, FOR DOWNTIME AND FOR BREACH... actual: {"parties":["eGain Communications Corporation","Eliance Corporation"],"effective_date":null,"term_months":12,"governing_law":"California","notice_period_days":30,"auto_renewal":true,"liability_cap":"an amount equal to the price paid for the Services out of which the claim arose"} - ct-014 expected: {"parties":["N2K Inc","Excite, Inc"],"effective_date":"1997-09-23","term_months":24,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"THE LIABILITY OF EXCITE FOR DAMAGES OR ALLEGED DAMAGES HEREUNDER, WHETHER IN CONTRACT, TORT OR ANY OTHER LEGA... actual: {"parties":["Excite, Inc.","N2K Inc."],"effective_date":"1997-09-23","term_months":24,"governing_law":"New York","notice_period_days":30,"auto_renewal":false,"liability_cap":"the amounts actually paid by Sponsor to Excite hereunder"} - ct-020 expected: {"parties":["Diplomat Ambassador Eyewear Group","KI Inc","Kathy Ireland, Inc"],"effective_date":"1995-08-01","term_months":54,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["Kathy Ireland, Inc.","The Sterling/Winters Co.","Diplomat Ambassador Eyewear Group"],"effective_date":"1995-08-24","term_months":54,"governing_law":"California","notice_period_days":20,"auto_renewal":false,"liability_cap":null} - ct-021 expected: {"parties":["GLOBAL ENERGY, INC","OXBOW CARBON & MINERALS LLC"],"effective_date":"2006-12-21","term_months":60,"governing_law":"New York","notice_period_days":null,"auto_renewal":true,"liability_cap":"Neither Party shalI be liable to the other Party in connection with this Agreement or the subject matter hereof for any... actual: {"parties":["Oxbow Carbon & Minerals LLC","Global Energy, Inc."],"effective_date":"2006-12-21","term_months":60,"governing_law":"New York","notice_period_days":30,"auto_renewal":true,"liability_cap":null} - ct-023 expected: {"parties":["Lucent Technologies Inc","mPhase Technologies Inc"],"effective_date":"2003-01-21","term_months":12,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"Neither Party shall be liable to the other for special, incidental, or consequential damages, even if such P... actual: {"parties":["Lucent Technologies Inc.","mPhase Technologies Inc."],"effective_date":"2003-01-21","term_months":12,"governing_law":"New York","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-024 expected: {"parties":["Hyatt Franchising Latin America, L.L.C","Playa Hotels & Resorts, B.V"],"effective_date":"2016-12-14","term_months":null,"governing_law":"Illinois","notice_period_days":null,"auto_renewal":false,"liability_cap":"In any action or proceeding between the Parties (including any arbitration proceeding) arising u... actual: {"parties":["Hyatt Franchising Latin America, L.L.C.","Playa Hotels & Resorts, B.V."],"effective_date":"2016-12-14","term_months":24,"governing_law":"Illinois","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-035 expected: {"parties":["HPIL ENERGYTECH Inc","GINARES GROUP AG"],"effective_date":"2015-01-05","term_months":12,"governing_law":"Nevada","notice_period_days":null,"auto_renewal":false,"liability_cap":"GINARES agrees that the sole and exclusive remedy for money damages related to this Agreement and the transactions contemplated h... actual: {"parties":["HPIL ENERGYTECH Inc.","GINARES GROUP AG"],"effective_date":"2015-01-05","term_months":12,"governing_law":"Nevada","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-036 expected: {"parties":["FEDERATED ADVISORY SERVICES COMPANY","FEDERATED INVESTMENT MANAGEMENT COMPANY"],"effective_date":"2004-01-01","governing_law":"Pennsylvania","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["FEDERATED INVESTMENT MANAGEMENT COMPANY","FEDERATED ADVISORY SERVICES COMPANY"],"effective_date":"2004-01-01","term_months":null,"governing_law":"Pennsylvania","notice_period_days":null,"auto_renewal":null,"liability_cap":null} - ct-037 expected: {"parties":["Torvec Inc","Phoenix Performance, LLC"],"effective_date":"2010-05-18","governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":"Upon termination of this Agreement, NEITHER PARTY SHALL BE LIABLE TO THE OTHER FOR ANY CONSEQUENTIAL, EXEMPLARY, SPECIAL, INCIDENTAL OR PUNITIVE ... actual: {"parties":["Phoenix Performance, LLC","Torvec Inc."],"effective_date":"2010-05-18","term_months":5,"governing_law":"New York","notice_period_days":null,"auto_renewal":false,"liability_cap":null} - ct-038 expected: {"parties":["National Football League Alumni - Northern California Chapter","National Football League Alumni, Inc","Food For Athletes, Inc"],"effective_date":"2017-11-01","term_months":36,"governing_law":"California","notice_period_days":null,"auto_renewal":false,"liability_cap":"In no event will NFLA be liable for any... actual: {"parties":["National Football League Alumni - Northern California Chapter","National Football League Alumni, Inc.","Food For Athletes, Inc.","Gridiron BioNutrients"],"effective_date":"2017-10-30","term_months":36,"governing_law":"California","notice_period_days":30,"auto_renewal":false,"liability_cap":"the remuneratio... - ct-039 expected: {"parties":["SQUARE TWO GOLF INC","KATHY WHITWORTH"],"effective_date":"2000-01-01","term_months":60,"governing_law":"New Jersey","notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["SQUARE TWO GOLF INC.","KATHY WHITWORTH"],"effective_date":"1999-10-13","term_months":60,"governing_law":"New Jersey","notice_period_days":30,"auto_renewal":false,"liability_cap":null} - ct-040 expected: {"parties":["SCANSOURCE DE MEXICO S. DE R.L. DE C.V","ScanSource Latin America, Inc","ScanSource, Inc","Zebra Brazil","Xplore Technologies Corporation of America","ScanSource Brazil Distribuidora de Technologias, Ltda","Zebra Technologies International, LLC","ScanSource Mexico"],"effective_date":"2019-02-04","term_mont... actual: {"parties":["Zebra Technologies International, LLC","Zebra Technologies do Brasil - Comércio de Produtos de Informåtica Ltda.","Xplore Technologies Corporation of America","ScanSource, Inc.","ScanSource Latin America, Inc.","ScanSource Brazil Distribuidora de Technologias, Ltda.","SCANSOURCE DE MEXICO S. DE R.L. DE C.V... - ct-041 expected: {"parties":["KIOSK CONCEPTS, INC"],"term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":false,"liability_cap":null} actual: {"parties":["KIOSK CONCEPTS, INC.","THE GRILLED CHEESE TRUCK, INC."],"effective_date":null,"term_months":null,"governing_law":null,"notice_period_days":null,"auto_renewal":null,"liability_cap":null} == TEXT / test 4 Answer with a citation / Claude Fable 5.1, as it ships (agent harness, no API) == (7 recorded, score 82.7, bar >= 80) - rag-en-007 expected: {"answer":"non-political Islam","citation":"p2"} actual: {"answer":"quietist/non-political Islam","citation":"p2"} - rag-en-008 expected: {"answer":"Astra","citation":"p3"} actual: {"answer":"Astra's satellites","citation":"p3"} - rag-en-016 expected: {"answer":"1685","citation":"p5"} actual: {"answer":"1598","citation":"p2"} - rag-de-028 expected: {"answer":"Das schottische Parlament","citation":"p4"} actual: {"answer":"vom schottischen Parlament","citation":"p4"} - rag-de-032 expected: {"answer":"Fertilität","citation":"p2"} actual: {"answer":"Steuerumverteilung","citation":"p2"} - rag-es-041 expected: {"answer":"Monasterio de Kumbum o Ta'er Shi, cerca de Xining","citation":"p4"} actual: {"answer":"al Monasterio de Kumbum (Ta'er Shi), cerca de Xining, 200 km más al oeste","citation":"p4"} - rag-es-044 expected: {"answer":"financiar","citation":"p2"} actual: {"answer":"financiando a los viajeros que regresaban con historias de sus descubrimientos","citation":"p2"} == TEXT / test 6 Inbox routing, short requests / Claude Fable 5.1, as it ships (agent harness, no API) == (2 recorded, score 97.9, bar >= 85) - tr-es-051 expected: {"category":"calendar"} actual: {"category":"news","urgency":"normal","department":"none"} - tr-fr-087 expected: {"category":"takeaway"} actual: {"category":"recommendation","urgency":"normal","department":"none"} == IMAGE / test V1 Scanned invoice / Qwen 3.5 35B-A3B FP8, as it ships == (2 recorded, score 98.4, bar >= 85) - v1h-003 expected: {"supplier_name":"Miller Inc","invoice_number":"INV76820400","issue_date":"2019-02-16","currency":"EUR","net_amount":693.67,"vat_amount":97.11,"vat_rate":null,"total_amount":790.78,"vat_rates":[20.0,5.0]} actual: {"supplier_name":"Miller Inc","supplier_vat_id":null,"invoice_number":"INV76820400","issue_date":"2019-02-16","due_date":"2019-03-02","currency":"EUR","net_amount":693.67,"vat_amount":96.11,"total_amount":790.78} - v1h-005 expected: {"supplier_name":"Rogers, Olson and Sanders","invoice_number":"CN-64837157","issue_date":"1998-04-23","due_date":null,"currency":"EUR","net_amount":-438.54,"vat_amount":-39.47,"vat_rate":9.0,"total_amount":-478.01,"credit_note":true} actual: {"supplier_name":"Rogers, Olson and Sanders","supplier_vat_id":null,"invoice_number":"CN-64837157","issue_date":"1998-04-23","due_date":"1998-05-07","currency":"EUR","net_amount":-438.54,"vat_amount":-39.47,"total_amount":-478.01} == IMAGE / test V2 Handwritten note / Qwen 3.5 35B-A3B FP8, as it ships == (17 recorded, score 15.0, bar >= 70) - v2-001 expected: {"text":"d° Jeannine 17 P f"} actual: 2° Jeanine 1.17 R. J. - v2-002 expected: {"text":"Decroix Jean 85 Eure m ch employé D R ?18377"} actual: Decroix Jean 75 Dare m. eh. employé - v2-003 expected: {"text":"Bellanger Edmond 65 Eure et L M ch Entrepreneur 14325"} actual: Bellanger Edmond 65 + Emelie M ch. Entrepreneur 1335 - v2-004 expected: {"text":"d° Jeanne 85 P M ep ménagère 21399"} actual: - 01 2 Jeanne 85 P M Ep. ménage 21.399 - v2-005 expected: {"text":"Motte Louis 90 Sre Infre m ch cuisinier 18372"} actual: Motte Louis 190 Pons 24 m. ch. cuisinier 18518 - v2-007 expected: {"text":"Jaumet Jean 89 O métallurgiste ?12207"} actual: Jauret Jean 85 X 0 mitallurgiste - v2-008 expected: {"text":"Marie 59 M Ep ménagère 21399"} actual: Marie 59 + -1- M E mingay 11349 - v2-010 expected: {"text":"Marcelle 98 P F o Steno Dactylo ?18576"} actual: Marcelle 98 P. F. o Stone Daddy 546 - v2-011 expected: {"text":"d° Roger 19 Orne f"} actual: 2° Roger / 19 / Orne / 8. - v2-012 expected: {"text":"Roussel Victor 66 P m ch bimbelotier 11161"} actual: Rowell Victor 166 / P. m. ch. burchlotier 11151 - v2-013 expected: {"text":"Haudion Marthe 66 aisne ch journalière 16352"} actual: Hauton Marthe 66 aime A journalier 16.352 - v2-014 expected: {"text":"Lacage Madeleine 04 H Gne ch E dessinat 14322"} actual: Lacaze Madeleine ou 2. fue ch E. delimal - v2-015 expected: {"text":"Reine 73 Nievre M Ep Néant 22108"} actual: Reine 73 Vire 4 Ep. Niant 99107 - v2-016 expected: {"text":"Liénard Auguste 59 Hte Saône V ch"} actual: Liénard Auguste 59 H. à laon V ch. - v2-017 expected: {"text":"Blanc Théophile 73 Savoie m ch gon mag Cabet?18352"} actual: Blanc Théophile 73 Savoie 1 m. ch. gme mag. 10556 cabot - v2-019 expected: {"text":"Escoffier Joseph 89 P M ch Menuisier 11159"} actual: Escoffier Joseph 89/8. Ach. Chemin 1-15g - v2-020 expected: {"text":"Marie 1917 f"} actual: , Marie 1917 -5- == IMAGE / test V3 Table as an image / Qwen 3.5 35B-A3B FP8, as it ships == (17 recorded, score 85.4, bar >= 85) - v3-001 expected: {"rows":[["Exposure time per week","People with dermatologic manifestations","Total people with exposure","Percent %"],["7–11","4","104","3.85"],["12–16","11","126","8.6"],["17–21","4","32","12.5"],["22–26","6","24","25"],["27–31","2","8","25"]]} actual: {"rows":[["Exposure time per week","People with dermatologic manifestations","Total people with exposure","Percent %"],["7-11","4","104","3.85"],["12-16","11","126","8.6"],["17-21","4","32","12.5"],["22-26","6","24","25"],["27-31","2","8","25"]]} - v3-002 expected: {"rows":[["taxonomic group","txid","# of sequences1"],["Metazoa","33208","862,420"],["Fungi","4751","184,282"],["Viridiplantae (plants and green algae)","33090","293,156"],["Non-green algae2","","21,889"],["Other Eukaryotes3","","49,732"],["Eubacteria (without Cyanobacteria)","2","1,386,089"],["Cyanobacteria","1117","9... actual: {"rows":[["taxonomic group","txid","# of sequences"],["Metazoa","33208","862,420"],["Fungi","4751","184,282"],["Viridiplantae (plants and green algae)","33090","293,156"],["Non-green algae","","21,889"],["Other Eukaryotes","","49,732"],["Eubacteria (without Cyanobacteria)","2","1,386,089"],["Cyanobacteria","1117","94,9... - v3-003 expected: {"rows":[["Stage","TNM","Modified Nevin"],["0","Tis N0 M0","In situ carcinoma"],["I","T1 N0 M0","In situ carcinoma"],["II","T2 N0 M0","Mucosal or muscular invasion"],["III","T1 N1 M0T2 N1 M0T3 N0 M0T3 N1 M0","Transmural direct liver invasion"],["IV A","T4 N0 M0T4 N1 M0","Lymph node metastasis"],["IV B","Any T N2 M0Any ... actual: {"rows":[["Stage","TNM","Modified Nevin"],["0","Tis N0 M0","In situ carcinoma"],["I","T1 N0 M0","In situ carcinoma"],["II","T2 N0 M0","Mucosal or muscular invasion"],["III","T3 N0 M0","Transmural direct liver invasion"],["","T3 N1 M0",""],["IV A","T4 N0 M0","Lymph node metastasis"],["","T4 N1 M0",""],["IV B","Any T N2 ... - v3-004 expected: {"rows":[["number sequences","NJ","BIONJ","Weighbor","HGT/FP","GME","BME"],["96 (500 nt)","9 (8)","8 (8)","10 (10)","12 (10)","10 (8)","10 (9)"],["500 (500 nt)","34 (37)","35 (39)","35 (36)","59 (29)","46 (33)","41 (39)"],["500 (1000 nt)","22 (19)","17 (23)","18 (22)","23 (28)","30 (20)","24 (20)"],["500 (2000 nt)","10... actual: {"rows":[["number sequences","NJ","BIONJ","Weighbor","HGT/FP","GME","BME"],["96 (500 nc)","9 (8)","8 (8)","10 (10)","12 (10)","10 (8)","10 (9)"],["500 (500 nc)","34 (37)","35 (39)","35 (36)","59 (29)","46 (33)","41 (39)"],["500 (1000 nc)","22 (19)","17 (23)","18 (22)","23 (28)","30 (20)","24 (20)"],["500 (2000 nc)","10... - v3-005 expected: {"rows":[["Genotype*","Perimeter (mm)**","%N2","n"],["kin-29(wk61)1, 2)","1.90 ± 0.15","73%","42"],["ctIs40 [pTG96(sur-5::gfp)]dbl-1(+)2","2.76 ± 0.12","106%","42"],["kin-29(wk61); ctIs40 [pTG96(sur-5::gfp)]1","1.84 ± 0.20","71%","51"],["sma-6(e1482) unc-4(e120)3","1.47 ± 0.13","57%","37"],["sma-6(e1482) unc-4(e120);ki... actual: {"rows":[["Genotype","Perimeter (mm)","%N2","n"],["kin-29(wk61)1, 2)","1.90 ± 0.15","73%","42"],["ctl#40 [pTG96/sur-5-gfp]-/+;ctl#40 [pTG96/sur-5-gfp]1","2.76 ± 0.12","106%","42"],["kin-29(wk61); ctl#40 [pTG96/sur-5-gfp]1","1.84 ± 0.20","71%","51"],["sma-6(e11482) unc-4(e120)3","1.47 ± 0.13","57%","37"],["sma-6(e11482)... - v3-006 expected: {"rows":[["","NCI-60 – HuFL","Brc 8k – HuFL","Brc8k – U95Av2","Lung – U95Av2"],["Total cDNA clones","9707","8820","8820","22691"],["Clones with both reads sequenced","6222","7015","7015","18645"],["Clones with predicted insert region","4639","6354","6354","14813"],["Total Probe-sets defined","1765","2403","3103","4597"... actual: {"rows":[["","NCI-60","Brc & U","Brc&U","Lung"],["Total cDNA clones","9707","8820","8820","22491"],["Clones with both reads sequenced","6222","7015","7015","18645"],["Clones with predicted insert region","4639","6354","6354","18413"],["Total Probe-sets defined","1765","2403","3103","4597"],["Total (perfect match) p... - v3-007 expected: {"rows":[["parameter","Squamous cell carcinoma (n = 229)","Adenocarcinoma (n = 150)","p-value"],["- Surgical complications (%)","","",""],["- General complications (%)","37.9","33.7","0.447"],["-Total complications (%)","40.7","28.0","0.012*"],["- Length of ICU – stay (d)","68.1","57.6","0.039*"],["- 30-day-moratlity (... actual: {"rows":[["parameter","Squamous cell carcinoma (n = 229)","Adenocarcinoma (n = 150)","p-value"],["- Surgical complications (%)","","",""],["- General complications (%)","37.9","33.7","0.447"],["-Total complications (%)","40.7","28.0","0.012*"],["- Length of ICU – stay (d)","68.1","57.6","0.039*"],["- 30-day-morality (%... - v3-008 expected: {"rows":[["Drug","Drug carrier","Targeting moiety","Inhibition of proliferation (%)"],["None","None","None","0.0 ± 8.2"],["FUdR","None","Anti-CD134","8.9 ± 8.7"],["FUdR-dP","Liposomes","None","6.3 ± 0.7"],["None","Liposomes","Anti-CD134","0.6 ± 1.1"],["FUdR-dP","Liposomes","Anti-CD134","31.0 ± 2.3*"]]} actual: {"rows":[["Drug","Drug carrier","Targeting moiety","Inhibition of proliferation (%)"],["None","None","None","0.0 ± 8.2"],["FdUR","None","Anti-CD134","8.9 ± 8.7"],["FdUR-dP","Liposomes","None","6.3 ± 0.7"],["None","Liposomes","Anti-CD134","0.6 ± 1.1"],["FdUR-dP","Liposomes","Anti-CD134","31.0 ± 2.3*"]]} - v3-009 expected: {"rows":[["Relation","Transitive","Symmetric","Reflexive","Antisymmetric"],["is_a","+","-","+","+"],["part_of","+","-","+","+"],["located_in","+","-","+","-"],["contained_in","-","-","-","-"],["adjacent_to","-","-","-","-"],["transformation_of","+","-","-","-"],["derives_ from","+","-","-","-"],["preceded_by","+","-","... actual: {"rows":[["Relation","Transitive","Symmetric","Reflexive","Antisymmetric"],["is_a","+","-","+","+"],["part_of","+","-","+","+"],["located_in","+","-","+","+"],["contained_in","+","-","-","-"],["adjacent_to","-","-","-","-"],["transformation_of","+","-","-","-"],["derives_from","-","-","-","-"],["preceded_by","+","-","-... - v3-012 expected: {"rows":[["Allel","amino acid nr.","Sequence","Reference"],["HLA-A*0201","65–73","RMPEAAPPV","[115,155]"],["HLA-B*4601","99–107","SQKTYQGSY","[117]"],["HLA-A2","103–111","YQGSYGFRL","[120]"],["HLA-A24","125–134","TYSPALNKMF","[156]"],["HLA-A2","139–147","KTCPVQLWV","[120,157]"],["HLA-A2.1","149–157","STPPPGTRV","[84,12... actual: {"rows":[["Allele","amino acid nr.","Sequence","Reference"],["HLA-A*0201","65–73","RMPEAPPV","[115,155]"],["HLA-B*4601","99–107","SQKTYQGSY","[17]"],["HLA-A2","103–111","YQGSYGIRL","[20]"],["HLA-A24","125–134","TYSPALNKFP","[56]"],["HLA-A2","139–147","KTCVPQLQWV","[20,157]"],["HLA-A2.1","149–157","STPPPRKTV","[84,124]"... - v3-013 expected: {"rows":[["","VAS","","","VRS","",""],["Pain etiology group","PA (%)","RP (95% CI)","RC (95% CI)","PA (%)","RP (95% CI)","RC (95% CI)"],["All, n = 81","20","0.01 (-0.03 to 0.04)","-0.02 (-0.08 to 0.05)","94","0.03 (-0.01 to 0.06)","0.04 (-0.01 to 0.08)"],["Chron/idiop, n = 30","20","-0.05 (-0.12 to 0.01)","-0.07 (-0.18... actual: {"rows":[["Pain etiology group","VAS","","","VRS","","",""],["","PA (%)","RP (95% CI)","RC (95% CI)","PA (%)","RP (95% CI)","RC (95% CI)",""],["All, n = 81","20","0.01","-0.02","94","0.03","0.04",""],["","","(−0.02 to 0.04)","(−0.08 to 0.05)","","(−0.01 to 0.06)","(−0.01 to 0.08)",""],["Chronic pain, n = 30","20","−0.0... - v3-014 expected: {"rows":[["","Factors strongly associated with LBP (OR > 1.2-)","Factors moderately associated with LBP (OR ≥ 1–1.2-)"],["Lack of fitness/Physical inactivity","Balague, 1999 [44]*Feuerstein, 1999[45] ****",""],["Smoking","Balague, 1999[44] *Feldmann, 1999[47] ***Levangie, 1999[48] ***Power, 2001[49] **","Leboeuf-Yde, 1... actual: {"rows":[["","Factors strongly associated with LBP (OR > 1.2)","Factors moderately associated with LBP (OR ≥ 1–1.2)"],["Lack of fitness/Physical inactivity","Balague, 1999[44] ◊◊◊\nFeuerstein, 1999[65] ◊◊◊\nBalague, 1999[44] ◊\nFeldstein, 1998[44] ◊◊\nLevangie, 1999[48] ◊◊\nPower, 2001[43] ◊◊",""],["Smoking","","Leboeu... - v3-016 expected: {"rows":[["Study","No.","Mean dose μg/kg/day (range)","Mean T1/2 (days)","Mean elimination rate constant (b)"],["Cox et al. 1989","55","(1.2–79.6)a","47.2","0.0147"],["Miettinen et al. 1971","6","0.3b","49.9","0.0142"],["Sherlock et al. 1984","20","1.6 (0.5–3.6)","50.2","0.0140"],["Kershaw et al. 1980","4","20.0 (18.1–... actual: {"rows":[["Study","No.","Mean dose μg/kg/day (range)","Mean T1/2 (days)","Mean elimination rate constant (β)"],["Cox et al. 1989","55","(1.2–79.6)a","47.2","0.0147"],["Miettinen et al. 1971","6","0.3b","49.9","0.0142"],["Sherlock et al. 1984","20","1.6 [0.5–3.6]","50.2","0.0140"],["Kershaw et al. 1980","4","20.0 (18.1–... - v3-017 expected: {"rows":[["Parameter","Current centrala tendency estimate","Pregnancy specific?","Third-trimester specific?","EPA central tendency estimate","Pregnancy specific?","Third-trimester specific?"],["R","1.7","Yes","Yes","1.0 (implicit)","No","No"],["b","0.0147 day−1 (47 days)","Yes","No","0.014 day−1 (50 days)","No","No"],[... actual: {"rows":[["Parameter","Current central tendency estimate","Pregnancy specific?","Third-trimester specific?","EPA central tendency estimate","Pregnancy specific?","Third-trimester specific?"],["R","1.7","Yes","Yes","1.0 (implicit)","No","No"],["b","0.0147 day-1 (47 days)","Yes","No","0.014 day-1 (50 days)","No","No"],["... - v3-018 expected: {"rows":[["Reference","Randomization","Sample size determination","Drop outs","Intention to treat analysis","Blinding","Allocation concealment"],["Kitchell 1963 [17]","Not described (nor stated)","Not described","33% of the treatment group upon cross-over","No","Allegedly participants blinded, but impossible to adequat... actual: {"rows":[["Reference","Randomization","Sample size determination","Drop outs","Intention to treat analysis","Blinding","Allocation concealment"],["Kitchell 1963 [17]","Not described (not stated)","Not described","33% of the treatment group upon cross-over","No","Allegedly participants blinded, but impossible to adequat... - v3-019 expected: {"rows":[["Age","Sex","n","Min","Max","Mean ± SD","Reference Range (95% CI)"],["1–16 Y","F","22","66","105","85.0 ± 10.8","63–106"],["","M","32","58","105","83.7 ± 11.1","63–106"],["Over16 Y","F","24","74","125","93.9 ± 13.6","67–121"],["","M","106","75","134","102.1 ± 12.2","79–126"]]} actual: {"rows":[["Age","Sex","n","Min","Max","Mean ± SD","Reference Range (95% CI)"],["I–16 Y","F","22","66","105","85.0 ± 10.8","63–106"],["","M","32","58","105","83.7 ± 11.1","63–106"],["Over16 Y","F","24","74","125","93.9 ± 13.6","67–121"],["","M","106","75","134","102.1 ± 12.2","79–126"]]} - v3-020 expected: {"rows":[["","Service Delivery State"],["Political arrangement","Democracy (multi-party), good governance"],["Institutional structure","Decentralised for service delivery"],["Technology achievable","Highest level – antiretroviral therapy"]]} actual: {"rows":[["Service Delivery State"],["Political arrangement","Democracy (multi-party), good governance"],["Institutional structure","Decentralised for service delivery"],["Technology achievable","Highest level – antiretroviral therapy"]]} == IMAGE / test V5 UI screenshot / Qwen 3.5 35B-A3B FP8, as it ships == (2 recorded, score 90.0, bar >= 80) - v5-001 expected: {"answers":["12 exercises","a total of 12 exercises","12","12 exercises in total"]} actual: 12 - v5-006 expected: {"answers":["\"+\"","press the \"+\" in the top-right corner","press the plus sign in the upper right","press the plus sign in the upper right corner","top-right \"+\"","press the plus sign in the top-right corner","\"+\" in the top-right corner","the \"+\" button"]} actual: {"answer":"top-right corner"} == IMAGE / test V7 No fabrication from an image / Qwen 3.5 35B-A3B FP8, as it ships == (20 recorded, score 2, bar <= 1) - v7-002 expected: {"present":false,"object":"backpack"} actual: null - v7-004 expected: {"present":false,"object":"car"} actual: null - v7-005 expected: {"present":true,"object":"skis"} actual: {"answer":"null"} - v7-006 expected: {"present":false,"object":"dog"} actual: null - v7-008 expected: {"present":false,"object":"car"} actual: {"answer":"black"} - v7-010 expected: {"present":false,"object":"dining table"} actual: null - v7-011 expected: {"present":true,"object":"Is there an umbrella in the imange?"} actual: {"answer":"null"} - v7-012 expected: {"present":false,"object":"handbag"} actual: null - v7-013 expected: {"present":true,"object":"person"} actual: null - v7-014 expected: {"present":false,"object":"dining table"} actual: null - v7-015 expected: {"present":true,"object":"bicycle"} actual: null - v7-016 expected: {"present":false,"object":"motorcycle"} actual: null - v7-018 expected: {"present":false,"object":"truck"} actual: {"answer":"white"} - v7-019 expected: {"present":true,"object":"person"} actual: null - v7-020 expected: {"present":false,"object":"dining table"} actual: null - v7-021 expected: {"present":false,"object":"vase"} actual: null - v7-022 expected: {"present":false,"object":"truck"} actual: null - v7-023 expected: {"present":false,"object":"bus"} actual: null - v7-024 expected: {"present":false,"object":"dining table"} actual: null - v7-025 expected: {"present":false,"object":"truck"} actual: null == IMAGE / test V2 Handwritten note / Qwen 3.5 35B-A3B FP8 + Unrote layer p1.4 == (17 recorded, score 15.0, bar >= 70) - v2-001 expected: {"text":"d° Jeannine 17 P f"} actual: 2° Jeannine 1.17 R. J. - v2-002 expected: {"text":"Decroix Jean 85 Eure m ch employé D R ?18377"} actual: Decroix Jean 75 Dare m. eh. employé - v2-003 expected: {"text":"Bellanger Edmond 65 Eure et L M ch Entrepreneur 14325"} actual: Bellanger Edmond 65 + Emelie M ch. Entrepreneur 1335 - v2-004 expected: {"text":"d° Jeanne 85 P M ep ménagère 21399"} actual: - 01 2 Jeanne 85 P M Ep. ménage 21.399 - v2-005 expected: {"text":"Motte Louis 90 Sre Infre m ch cuisinier 18372"} actual: Motte Louis 190 Pons Nf. m. ch. cuisinier 1851 - v2-007 expected: {"text":"Jaumet Jean 89 O métallurgiste ?12207"} actual: Jauret Jean 8f x 0 mitallurgiste - v2-008 expected: {"text":"Marie 59 M Ep ménagère 21399"} actual: Marie 59 + -1- M E mingay 11349 - v2-010 expected: {"text":"Marcelle 98 P F o Steno Dactylo ?18576"} actual: Marcelle 98 P. F. o Stone Daddy 546 - v2-011 expected: {"text":"d° Roger 19 Orne f"} actual: 2° Roger / 19 / Orne / 8. - v2-012 expected: {"text":"Roussel Victor 66 P m ch bimbelotier 11161"} actual: Rowell Victor 166 1 P. m. ch. bachelier 11151 - v2-013 expected: {"text":"Haudion Marthe 66 aisne ch journalière 16352"} actual: Hauton Marthe 66 aime A journalier 16.352 - v2-014 expected: {"text":"Lacage Madeleine 04 H Gne ch E dessinat 14322"} actual: Lacaze Madeleine ou 2. fme ch E. delimal - v2-015 expected: {"text":"Reine 73 Nievre M Ep Néant 22108"} actual: Reine 73 Nive 4 Ep. Niant 92107 - v2-016 expected: {"text":"Liénard Auguste 59 Hte Saône V ch"} actual: Liénard Auguste 59 H. à laon V ch. - v2-017 expected: {"text":"Blanc Théophile 73 Savoie m ch gon mag Cabet?18352"} actual: Blanc Théophile 73 Savoie 1 m. ch. gme mag. 10556 cabot - v2-019 expected: {"text":"Escoffier Joseph 89 P M ch Menuisier 11159"} actual: Escoffier Joseph 89/8. Ach. Chemin M-159 - v2-020 expected: {"text":"Marie 1917 f"} actual: , Marie 1917 -5- == IMAGE / test V3 Table as an image / Qwen 3.5 35B-A3B FP8 + Unrote layer p1.4 == (12 recorded, score 89.4, bar >= 85) - v3-002 expected: {"rows":[["taxonomic group","txid","# of sequences1"],["Metazoa","33208","862,420"],["Fungi","4751","184,282"],["Viridiplantae (plants and green algae)","33090","293,156"],["Non-green algae2","","21,889"],["Other Eukaryotes3","","49,732"],["Eubacteria (without Cyanobacteria)","2","1,386,089"],["Cyanobacteria","1117","9... actual: {"rows":[["taxonomic group","txid","# of sequences¹"],["Metazoa","33208","862,420"],["Fungi","4751","184,282"],["Viridiplantae (plants and green algae)","33090","293,156"],["Non-green algae²","","21,889"],["Other Eukaryotes³","","49,732"],["Eubacteria (without Cyanobacteria)","2","1,386,089"],["Cyanobacteria","1117","9... - v3-003 expected: {"rows":[["Stage","TNM","Modified Nevin"],["0","Tis N0 M0","In situ carcinoma"],["I","T1 N0 M0","In situ carcinoma"],["II","T2 N0 M0","Mucosal or muscular invasion"],["III","T1 N1 M0T2 N1 M0T3 N0 M0T3 N1 M0","Transmural direct liver invasion"],["IV A","T4 N0 M0T4 N1 M0","Lymph node metastasis"],["IV B","Any T N2 M0Any ... actual: {"rows":[["Stage","TNM","Modified Nevin"],["0","Tis N0 M0","In situ carcinoma"],["I","T1 N0 M0","In situ carcinoma"],["II","T2 N0 M0","Mucosal or muscular invasion"],["III","T1 N1 M0","Transmural direct liver invasion"],["","T2 N1 M0",""],["","T3 N0 M0",""],["","T3 N1 M0",""],["IV A","T4 N0 M0","Lymph node metastasis"]... - v3-005 expected: {"rows":[["Genotype*","Perimeter (mm)**","%N2","n"],["kin-29(wk61)1, 2)","1.90 ± 0.15","73%","42"],["ctIs40 [pTG96(sur-5::gfp)]dbl-1(+)2","2.76 ± 0.12","106%","42"],["kin-29(wk61); ctIs40 [pTG96(sur-5::gfp)]1","1.84 ± 0.20","71%","51"],["sma-6(e1482) unc-4(e120)3","1.47 ± 0.13","57%","37"],["sma-6(e1482) unc-4(e120);ki... actual: {"rows":[["Genotype*","Perimeter (mm)**","%N2","n"],["kin-29(wk61)1, 2)","1.90 ± 0.15","73%","42"],["ctl-40 [pTG96(sur-5::gfp)]dbl-1(+)2","2.76 ± 0.12","106%","42"],["kin-29(wk61); ctl-40 [pTG96(sur-5::gfp)]1","1.84 ± 0.20","71%","51"],["smo-6(e1482) unc-4(e120)3","1.47 ± 0.13","57%","37"],["smo-6(e1482) unc-4(e120);ki... - v3-006 expected: {"rows":[["","NCI-60 – HuFL","Brc 8k – HuFL","Brc8k – U95Av2","Lung – U95Av2"],["Total cDNA clones","9707","8820","8820","22691"],["Clones with both reads sequenced","6222","7015","7015","18645"],["Clones with predicted insert region","4639","6354","6354","14813"],["Total Probe-sets defined","1765","2403","3103","4597"... actual: {"rows":[["","NCI-60 - HuFL","Brc 8k - HuFL","Brc8k - U95Av2","Lung - U95Av2"],["Total cDNA clones","9707","8820","8820","22691"],["Clones with both reads sequenced","6222","7015","7015","18645"],["Clones with predicted insert region","4639","6354","6354","14813"],["Total Probe-sets defined","1765","2403","3103","4597"... - v3-007 expected: {"rows":[["parameter","Squamous cell carcinoma (n = 229)","Adenocarcinoma (n = 150)","p-value"],["- Surgical complications (%)","","",""],["- General complications (%)","37.9","33.7","0.447"],["-Total complications (%)","40.7","28.0","0.012*"],["- Length of ICU – stay (d)","68.1","57.6","0.039*"],["- 30-day-moratlity (... actual: {"rows":[["parameter","Squamous cell carcinoma (n = 229)","Adenocarcinoma (n = 150)","p-value"],["- Surgical complications (%)","","",""],["- General complications (%)","37.9","33.7","0.447"],["- Total complications (%)","40.7","28.0","0.012*"],["- Length of ICU – stay (d)","68.1","57.6","0.039*"],["- 30-day-moratlity ... - v3-009 expected: {"rows":[["Relation","Transitive","Symmetric","Reflexive","Antisymmetric"],["is_a","+","-","+","+"],["part_of","+","-","+","+"],["located_in","+","-","+","-"],["contained_in","-","-","-","-"],["adjacent_to","-","-","-","-"],["transformation_of","+","-","-","-"],["derives_ from","+","-","-","-"],["preceded_by","+","-","... actual: {"rows":[["Relation","Transitive","Symmetric","Reflexive","Antisymmetric"],["is_a","+","-","+","+"],["part_of","+","-","+","+"],["located_in","+","-","+","-"],["contained_in","-","-","-","-"],["adjacent_to","-","-","-","-"],["transformation_of","+","-","-","-"],["derives_from","+","-","-","-"],["preceded_by","+","-","-... - v3-012 expected: {"rows":[["Allel","amino acid nr.","Sequence","Reference"],["HLA-A*0201","65–73","RMPEAAPPV","[115,155]"],["HLA-B*4601","99–107","SQKTYQGSY","[117]"],["HLA-A2","103–111","YQGSYGFRL","[120]"],["HLA-A24","125–134","TYSPALNKMF","[156]"],["HLA-A2","139–147","KTCPVQLWV","[120,157]"],["HLA-A2.1","149–157","STPPPGTRV","[84,12... actual: {"rows":[["Allel","amino acid nr.","Sequence","Reference"],["HLA-A*0201","65–73","RMPEAAPPV","[115,155]"],["HLA-B*4601","99–107","SQKTYQGSY","[117]"],["HLA-A2","103–111","YQGSYGRL","[120]"],["HLA-A24","125–134","TYSPALNKM","[156]"],["HLA-A2","139–147","KTCPVQLWV","[120,157]"],["HLA-A2.1","149–157","STPPPGTRV","[84,124]... - v3-013 expected: {"rows":[["","VAS","","","VRS","",""],["Pain etiology group","PA (%)","RP (95% CI)","RC (95% CI)","PA (%)","RP (95% CI)","RC (95% CI)"],["All, n = 81","20","0.01 (-0.03 to 0.04)","-0.02 (-0.08 to 0.05)","94","0.03 (-0.01 to 0.06)","0.04 (-0.01 to 0.08)"],["Chron/idiop, n = 30","20","-0.05 (-0.12 to 0.01)","-0.07 (-0.18... actual: {"rows":[["Pain etiology group","VAS\nPA (%)","RP (95% CI)","RC (95% CI)","VRS\nPA (%)","RP (95% CI)","RC (95% CI)"],["All, n = 81","20","0.01","-0.02","94","0.03","0.04"],["","","(-0.03 to 0.04)","(-0.08 to 0.05)","","(-0.01 to 0.06)","(-0.01 to 0.08)"],["Chron/idiop., n = 30","20","-0.05","-0.07","97","0.02","-0.01"]... - v3-016 expected: {"rows":[["Study","No.","Mean dose μg/kg/day (range)","Mean T1/2 (days)","Mean elimination rate constant (b)"],["Cox et al. 1989","55","(1.2–79.6)a","47.2","0.0147"],["Miettinen et al. 1971","6","0.3b","49.9","0.0142"],["Sherlock et al. 1984","20","1.6 (0.5–3.6)","50.2","0.0140"],["Kershaw et al. 1980","4","20.0 (18.1–... actual: {"rows":[["Study","No.","Mean dose\nμg/kg/day [range]","Mean T1/2 [days]","Mean elimination\nrate constant (b)"],["Cox et al. 1989","55","(1.2–79.6)a","47.2","0.0147"],["Miettinen et al. 1971","6","0.3b","49.9","0.0142"],["Sherlock et al. 1984","20","1.6 [0.5–3.6]","50.2","0.0140"],["Kershaw et al. 1980","4","20.0 (18.... - v3-017 expected: {"rows":[["Parameter","Current centrala tendency estimate","Pregnancy specific?","Third-trimester specific?","EPA central tendency estimate","Pregnancy specific?","Third-trimester specific?"],["R","1.7","Yes","Yes","1.0 (implicit)","No","No"],["b","0.0147 day−1 (47 days)","Yes","No","0.014 day−1 (50 days)","No","No"],[... actual: {"rows":[["Parameter","Current central tendency estimate","Pregnancy specific?","Third-trimester specific?","EPA central tendency estimate","Pregnancy specific?","Third-trimester specific?"],["R","1.7","Yes","Yes","1.0 (implicit)","No","No"],["b","0.0147 day⁻¹ (47 days)","Yes","No","0.014 day⁻¹ (50 days)","No","No"],["... - v3-018 expected: {"rows":[["Reference","Randomization","Sample size determination","Drop outs","Intention to treat analysis","Blinding","Allocation concealment"],["Kitchell 1963 [17]","Not described (nor stated)","Not described","33% of the treatment group upon cross-over","No","Allegedly participants blinded, but impossible to adequat... actual: {"rows":[["Reference","Randomization","Sample size determination","Drop outs","Intention to treat analysis","Blinding","Allocation concealment"],["Kitchell 1963 [17]","Not described (nor stated)","Not described","33% of the treatment group upon cross-over","No","Allegedly participants blinded, but impossible to adequat... - v3-020 expected: {"rows":[["","Service Delivery State"],["Political arrangement","Democracy (multi-party), good governance"],["Institutional structure","Decentralised for service delivery"],["Technology achievable","Highest level – antiretroviral therapy"]]} actual: {"rows":[["Service Delivery State"],["Political arrangement","Democracy (multi-party), good governance"],["Institutional structure","Decentralised for service delivery"],["Technology achievable","Highest level – antiretroviral therapy"]]} == IMAGE / test V4 Photo moderation / Qwen 3.5 35B-A3B FP8 + Unrote layer p1.4 == (1 recorded, score 99.0, bar >= 85) - v4-086 expected: {"decision":"review","labels":["Cocktail"]} actual: {"decision":"allowed","reason":"The policy allows non-alcoholic drinks and does not list these items as a violation or review case."} == IMAGE / test V5 UI screenshot / Qwen 3.5 35B-A3B FP8 + Unrote layer p1.4 == (1 recorded, score 95.0, bar >= 80) - v5-006 expected: {"answers":["\"+\"","press the \"+\" in the top-right corner","press the plus sign in the upper right","press the plus sign in the upper right corner","top-right \"+\"","press the plus sign in the top-right corner","\"+\" in the top-right corner","the \"+\" button"]} actual: {"answer":"top-right corner"} == IMAGE / test V7 No fabrication from an image / Qwen 3.5 35B-A3B FP8 + Unrote layer p1.4 == (3 recorded, score 1, bar <= 1) - v7-005 expected: {"present":true,"object":"skis"} actual: {"answer":null} - v7-008 expected: {"present":false,"object":"car"} actual: {"answer":"black"} - v7-017 expected: {"present":true,"object":"car"} actual: {"answer":null} == IMAGE / test V2 Handwritten note / Gemini 3.8 Flash, as it ships == (12 recorded, score 40.0, bar >= 70) - v2-002 expected: {"text":"Decroix Jean 85 Eure m ch employé D R ?18377"} actual: Lecroix Jean 85 Eure 1 m. ch. employé D.K. - v2-003 expected: {"text":"Bellanger Edmond 65 Eure et L M ch Entrepreneur 14325"} actual: Bellanger | Edmond | 69 | Eure et L | | M | Ch. - v2-004 expected: {"text":"d° Jeanne 85 P M ep ménagère 21399"} actual: - d° - | Jeanne | 85 | f | | M | Esp. | - mén - v2-005 expected: {"text":"Motte Louis 90 Sre Infre m ch cuisinier 18372"} actual: Motte Louis 90 R.on Inf.re m. ch. cuisinier 1 - v2-010 expected: {"text":"Marcelle 98 P F o Steno Dactylo ?18576"} actual: Marcelle 98 P. F. - o Steno Dactylo 18576 - v2-012 expected: {"text":"Roussel Victor 66 P m ch bimbelotier 11161"} actual: Rouard Victor 66 F. m. ch. bimbelot - v2-014 expected: {"text":"Lacage Madeleine 04 H Gne ch E dessinat 14322"} actual: Lacaze Madeleine 04 d. fille ch E. delhial - v2-015 expected: {"text":"Reine 73 Nievre M Ep Néant 22108"} actual: | Reine | 79 | Nièvre | M | Ep. | - v2-016 expected: {"text":"Liénard Auguste 59 Hte Saône V ch"} actual: Liénard | Auguste | 59 | H.te Saône | V - v2-017 expected: {"text":"Blanc Théophile 73 Savoie m ch gon mag Cabet?18352"} actual: Blanc Théophile 73 Savoie 1 m. ch. g^{d} mag. - v2-019 expected: {"text":"Escoffier Joseph 89 P M ch Menuisier 11159"} actual: Escoffier Joseph 891 G. le ch. Menuisier - v2-020 expected: {"text":"Marie 1917 f"} actual: " Marie 1910 - 5 - f == IMAGE / test V3 Table as an image / Gemini 3.8 Flash, as it ships == (12 recorded, score 78.6, bar >= 85) - v3-001 expected: {"rows":[["Exposure time per week","People with dermatologic manifestations","Total people with exposure","Percent %"],["7–11","4","104","3.85"],["12–16","11","126","8.6"],["17–21","4","32","12.5"],["22–26","6","24","25"],["27–31","2","8","25"]]} actual: {"rows":[["Exposure time per week","People with dermatologic manifestations","Total people with exposure","Percent %"],["7-11","4","104","3.85"],["12-16","11","126","8.6"],["17-21","4","32","12.5"],["22-26","6","24","25"],["27-31","2","8","25"]]} - v3-002 expected: {"rows":[["taxonomic group","txid","# of sequences1"],["Metazoa","33208","862,420"],["Fungi","4751","184,282"],["Viridiplantae (plants and green algae)","33090","293,156"],["Non-green algae2","","21,889"],["Other Eukaryotes3","","49,732"],["Eubacteria (without Cyanobacteria)","2","1,386,089"],["Cyanobacteria","1117","9... actual: {"rows":[["taxonomic group","txid","# of\nsequences¹"],["Metazoa","33208","862,420"],["Fungi","4751","184,282"],["Viridiplantae (plants and green algae)","33090","293,156"],["Non-green algae²","","21,889"],["Other Eukaryotes³","","49,732"],["Eubacteria (without Cyanobacteria)","2","1,386,089"],["Cyanobacteria","1117","... - v3-006 expected: {"rows":[["","NCI-60 – HuFL","Brc 8k – HuFL","Brc8k – U95Av2","Lung – U95Av2"],["Total cDNA clones","9707","8820","8820","22691"],["Clones with both reads sequenced","6222","7015","7015","18645"],["Clones with predicted insert region","4639","6354","6354","14813"],["Total Probe-sets defined","1765","2403","3103","4597"... actual: {"rows":[["","NCI-60 - HuFL","Brc 8k - HuFL","Brc8k - U95Av2","Lung - U95Av2"],["Total cDNA clones","9707","8820","8820","22691"],["Clones with both reads sequenced","6222","7015","7015","18645"],["Clones with predicted insert region","4639","6354","6354","14813"],["Total Probe-sets defined","1765","2403","3103","4597"... - v3-007 expected: {"rows":[["parameter","Squamous cell carcinoma (n = 229)","Adenocarcinoma (n = 150)","p-value"],["- Surgical complications (%)","","",""],["- General complications (%)","37.9","33.7","0.447"],["-Total complications (%)","40.7","28.0","0.012*"],["- Length of ICU – stay (d)","68.1","57.6","0.039*"],["- 30-day-moratlity (... actual: ```json {"rows": [["parameter", "Squamous cell carcinoma (n = 229)", "Adenocarcinoma (n = 150)", "p-value"], ["- Surgical complications (%)", "37.9", "33.7", "0.447"], ["- General complications (%)", "40.7", "28. - v3-009 expected: {"rows":[["Relation","Transitive","Symmetric","Reflexive","Antisymmetric"],["is_a","+","-","+","+"],["part_of","+","-","+","+"],["located_in","+","-","+","-"],["contained_in","-","-","-","-"],["adjacent_to","-","-","-","-"],["transformation_of","+","-","-","-"],["derives_ from","+","-","-","-"],["preceded_by","+","-","... actual: {"rows":[["Relation","Transitive","Symmetric","Reflexive","Antisymmetric"],["is_a","+","-","+","+"],["part_of","+","-","+","+"],["located_in","+","-","+","-"],["contained_in","-","-","-","-"],["adjacent_to","-","-","-","-"],["transformation_of","+","-","-","-"],["derives_from","+","-","-","-"],["preceded_by","+","-","-... - v3-010 expected: {"rows":[["","Healthy neonates (n = 48)","CLD infants (n = 48)"],["Birth weight (g)","3280 (1610 – 4670)","890*** (450 – 3860)"],["Gestational age (weeks)","39 (34 – 41)","27*** (24 – 34)"],["Age (days)","7 (3 – 12)","86*** (33 – 125)"],["Postconceptional age (weeks)","39.6 (35.6 – 42.3)","38.7 (36.0 – 42.3)"],["Body w... actual: - v3-012 expected: {"rows":[["Allel","amino acid nr.","Sequence","Reference"],["HLA-A*0201","65–73","RMPEAAPPV","[115,155]"],["HLA-B*4601","99–107","SQKTYQGSY","[117]"],["HLA-A2","103–111","YQGSYGFRL","[120]"],["HLA-A24","125–134","TYSPALNKMF","[156]"],["HLA-A2","139–147","KTCPVQLWV","[120,157]"],["HLA-A2.1","149–157","STPPPGTRV","[84,12... actual: {"rows":[["Allel","amino acid nr.","Sequence","Reference"],["HLA-A*0201","65-73","RMPEAAPPV","[115,155]"],["HLA-B*4601","99-107","SQKTYQGSY","[117]"],["HLA-A2","103-111","YQGSYGFRL","[120]"],["HLA-A24","125-134","TYSPALNKMF","[156]"],["HLA-A2","139-147","KTCPVQLWV","[120,157]"],["HLA-A2.1","149-157","STPPPGTRV","[84,12... - v3-014 expected: {"rows":[["","Factors strongly associated with LBP (OR > 1.2-)","Factors moderately associated with LBP (OR ≥ 1–1.2-)"],["Lack of fitness/Physical inactivity","Balague, 1999 [44]*Feuerstein, 1999[45] ****",""],["Smoking","Balague, 1999[44] *Feldmann, 1999[47] ***Levangie, 1999[48] ***Power, 2001[49] **","Leboeuf-Yde, 1... actual: {"rows":[["","Factors strongly associated with LBP (OR > 1.2-)","Factors moderately associated with LBP (OR ≥ 1-1.2-)"],["Lack of fitness/Physical inactivity","Balague, 1999 [44]*\nFeuerstein, 1999[45] ****",""],["Smoking","Balague, 1999[44] *\nFeldmann, 1999[47] ***\nLevangie, 1999[48] ***\nPower, 2001[49] **","Leboeu... - v3-016 expected: {"rows":[["Study","No.","Mean dose μg/kg/day (range)","Mean T1/2 (days)","Mean elimination rate constant (b)"],["Cox et al. 1989","55","(1.2–79.6)a","47.2","0.0147"],["Miettinen et al. 1971","6","0.3b","49.9","0.0142"],["Sherlock et al. 1984","20","1.6 (0.5–3.6)","50.2","0.0140"],["Kershaw et al. 1980","4","20.0 (18.1–... actual: {"rows":[["Study","No.","Mean dose μg/kg/day (range)","Mean T₁/₂ (days)","Mean elimination rate constant (b)"],["Cox et al. 1989","55","(1.2–79.6)ᵃ","47.2","0.0147"],["Miettinen et al. 1971","6","0.3ᵇ","49.9","0.0142"],["Sherlock et al. 1984","20","1.6 (0.5–3.6)","50.2","0.0140"],["Kershaw et al. 1980","4","20.0 (18.1–... - v3-017 expected: {"rows":[["Parameter","Current centrala tendency estimate","Pregnancy specific?","Third-trimester specific?","EPA central tendency estimate","Pregnancy specific?","Third-trimester specific?"],["R","1.7","Yes","Yes","1.0 (implicit)","No","No"],["b","0.0147 day−1 (47 days)","Yes","No","0.014 day−1 (50 days)","No","No"],[... actual: {"rows":[["Parameter","Current centralª\ntendency estimate","Pregnancy\nspecific?","Third-trimester\nspecific?","EPA central\ntendency estimate","Pregnancy\nspecific?","Third-trimester\nspecific?"],["R","1.7","Yes","Yes","1.0 (implicit)","No","No"],["b","0.0147 day⁻¹\n(47 days)","Yes","No","0.014 day⁻¹\n(50 days)","No"... - v3-018 expected: {"rows":[["Reference","Randomization","Sample size determination","Drop outs","Intention to treat analysis","Blinding","Allocation concealment"],["Kitchell 1963 [17]","Not described (nor stated)","Not described","33% of the treatment group upon cross-over","No","Allegedly participants blinded, but impossible to adequat... actual: - v3-020 expected: {"rows":[["","Service Delivery State"],["Political arrangement","Democracy (multi-party), good governance"],["Institutional structure","Decentralised for service delivery"],["Technology achievable","Highest level – antiretroviral therapy"]]} actual: {"rows":[["","Service Delivery State"],["Political arrangement","Democracy (multi-party), good governance"],["Institutional structure","Decentralised for service delivery"],["Technology achievable","Highest level - antiretroviral therapy"]]} == IMAGE / test V5 UI screenshot / Gemini 3.8 Flash, as it ships == (2 recorded, score 90.0, bar >= 80) - v5-006 expected: {"answers":["\"+\"","press the \"+\" in the top-right corner","press the plus sign in the upper right","press the plus sign in the upper right corner","top-right \"+\"","press the plus sign in the top-right corner","\"+\" in the top-right corner","the \"+\" button"]} actual: {"answer":"top-right corner"} - v5-015 expected: {"answers":["on","enabled"]} actual: {"answer":"checked"} == IMAGE / test V7 No fabrication from an image / Gemini 3.8 Flash, as it ships == (9 recorded, score 4, bar <= 1) - v7-005 expected: {"present":true,"object":"skis"} actual: {"answer":null} - v7-008 expected: {"present":false,"object":"car"} actual: {"answer":"black"} - v7-009 expected: {"present":true,"object":"person"} actual: {"answer":null} - v7-013 expected: {"present":true,"object":"person"} actual: {"answer":null} - v7-017 expected: {"present":true,"object":"car"} actual: {"answer":null} - v7-019 expected: {"present":true,"object":"person"} actual: {"answer":null} - v7-023 expected: {"present":false,"object":"bus"} actual: {"answer":"yellow"} - v7-025 expected: {"present":false,"object":"truck"} actual: {"answer":"yellow"} - v7-030 expected: {"present":false,"object":"cup"} actual: {"answer":"red"} == IMAGE / test V1 Scanned invoice / Claude Fable 5.1, as it ships (agent harness, no API) == (1 recorded, score 87.5, bar >= 85) - v1-001 expected: {"supplier_name":"Montgomery, Harrison and Vaughan","invoice_number":"6Y6M7d-731","issue_date":"1996-03-01","due_date":"2019-08-11","total_amount":967.7,"currency":"EUR","net_amount":942.26,"vat_amount":47.49,"discount_amount":22.05} actual: {"supplier_name":"Montgomery, Harrison and Vaughan","supplier_vat_id":null,"invoice_number":"6Y6M7d-731","issue_date":"1996-03-01","due_date":"2019-08-11","currency":"EUR","net_amount":920.21,"vat_amount":47.49,"total_amount":967.7} == IMAGE / test V2 Handwritten note / Claude Fable 5.1, as it ships (agent harness, no API) == (9 recorded, score 55.0, bar >= 70) - v2-001 expected: {"text":"d° Jeannine 17 P f"} actual: d° Jeanine 17 P. f. - v2-002 expected: {"text":"Decroix Jean 85 Eure m ch employé D R ?18377"} actual: Decroix Jean 85 Eure m. ch. employé - v2-005 expected: {"text":"Motte Louis 90 Sre Infre m ch cuisinier 18372"} actual: Motte Louis 90 S.ne Inf.re m. ch. cuisinier 18512 - v2-012 expected: {"text":"Roussel Victor 66 P m ch bimbelotier 11161"} actual: Roussel Victor 66 P. m. ch. bimbelotier 11151 - v2-014 expected: {"text":"Lacage Madeleine 04 H Gne ch E dessinat 14322"} actual: Lacaze Madeleine 04 D. Gue ch E. dessinat - v2-015 expected: {"text":"Reine 73 Nievre M Ep Néant 22108"} actual: Reine 79 Nièvre M Ep. Néant 2267 - v2-017 expected: {"text":"Blanc Théophile 73 Savoie m ch gon mag Cabet?18352"} actual: Blanc Théophile 73 Savoie m. ch. gon mag. Tabet - v2-019 expected: {"text":"Escoffier Joseph 89 P M ch Menuisier 11159"} actual: Escoffier Joseph 89 P. Vf ch. Menuisier 11.159 - v2-020 expected: {"text":"Marie 1917 f"} actual: Marie 1917 - s - == IMAGE / test V3 Table as an image / Claude Fable 5.1, as it ships (agent harness, no API) == (13 recorded, score 93.8, bar >= 85) - v3-002 expected: {"rows":[["taxonomic group","txid","# of sequences1"],["Metazoa","33208","862,420"],["Fungi","4751","184,282"],["Viridiplantae (plants and green algae)","33090","293,156"],["Non-green algae2","","21,889"],["Other Eukaryotes3","","49,732"],["Eubacteria (without Cyanobacteria)","2","1,386,089"],["Cyanobacteria","1117","9... actual: {"rows":[["taxonomic group","txid","# of sequences†"],["Metazoa","33208","862,420"],["Fungi","4751","184,282"],["Viridiplantae (plants and green algae)","33090","293,156"],["Non-green algae2","","21,889"],["Other Eukaryotes3","","49,732"],["Eubacteria (without Cyanobacteria)","2","1,386,089"],["Cyanobacteria","1117","9... - v3-003 expected: {"rows":[["Stage","TNM","Modified Nevin"],["0","Tis N0 M0","In situ carcinoma"],["I","T1 N0 M0","In situ carcinoma"],["II","T2 N0 M0","Mucosal or muscular invasion"],["III","T1 N1 M0T2 N1 M0T3 N0 M0T3 N1 M0","Transmural direct liver invasion"],["IV A","T4 N0 M0T4 N1 M0","Lymph node metastasis"],["IV B","Any T N2 M0Any ... actual: {"rows":[["Stage","TNM","Modified Nevin"],["0","Tis N0 M0","In situ carcinoma"],["I","T1 N0 M0","In situ carcinoma"],["II","T2 N0 M0","Mucosal or muscular invasion"],["III","T1 N1 M0","Transmural direct liver invasion"],["","T2 N1 M0",""],["","T3 N0 M0",""],["","T3 N1 M0",""],["IV A","T4 N0 M0","Lymph node metastasis"]... - v3-004 expected: {"rows":[["number sequences","NJ","BIONJ","Weighbor","HGT/FP","GME","BME"],["96 (500 nt)","9 (8)","8 (8)","10 (10)","12 (10)","10 (8)","10 (9)"],["500 (500 nt)","34 (37)","35 (39)","35 (36)","59 (29)","46 (33)","41 (39)"],["500 (1000 nt)","22 (19)","17 (23)","18 (22)","23 (28)","30 (20)","24 (20)"],["500 (2000 nt)","10... actual: {"rows":[["number sequences","NJ","BIONJ","Weighbor","HGT/FP","GME","BME"],["96 (500 nt)","9 (8)","8 (8)","10 (10)","12 (10)","10 (8)","10 (9)"],["100 (500 nt)","34 (37)","35 (39)","35 (36)","59 (29)","46 (33)","41 (39)"],["500 (1000 nt)","22 (19)","17 (23)","18 (18)","26 (18)","20 (18)","20 (20)"],["500 (2000 nt)","10... - v3-005 expected: {"rows":[["Genotype*","Perimeter (mm)**","%N2","n"],["kin-29(wk61)1, 2)","1.90 ± 0.15","73%","42"],["ctIs40 [pTG96(sur-5::gfp)]dbl-1(+)2","2.76 ± 0.12","106%","42"],["kin-29(wk61); ctIs40 [pTG96(sur-5::gfp)]1","1.84 ± 0.20","71%","51"],["sma-6(e1482) unc-4(e120)3","1.47 ± 0.13","57%","37"],["sma-6(e1482) unc-4(e120);ki... actual: {"rows":[["Genotype*","Perimeter (mm)**","%N2","n"],["kin-29(wk61)1, 2)","1.90 ± 0.15","73%","42"],["ctls40 [pTG96(sur-5::gfp)];dbl-1(+)2","2.76 ± 0.12","106%","42"],["kin-29(wk61); ctls40 [pTG96(sur-5::gfp)]1","1.84 ± 0.20","71%","51"],["smo-6(e1482) unc-4(e120)3","1.47 ± 0.13","57%","37"],["smo-6(e1482) unc-4(e120);k... - v3-006 expected: {"rows":[["","NCI-60 – HuFL","Brc 8k – HuFL","Brc8k – U95Av2","Lung – U95Av2"],["Total cDNA clones","9707","8820","8820","22691"],["Clones with both reads sequenced","6222","7015","7015","18645"],["Clones with predicted insert region","4639","6354","6354","14813"],["Total Probe-sets defined","1765","2403","3103","4597"... actual: {"rows":[["","NCI-60 – HuFL","Brc 8k – HuFL","Brc8k – U95Av2","Lung – U95Av2"],["Total cDNA clones","9707","8820","8820","22691"],["Clones with both reads sequenced","6222","7015","7015","18645"],["Clones with predicted insert region","4639","6354","6354","11813"],["Total Probe-sets defined","1765","2403","3103","4597"... - v3-007 expected: {"rows":[["parameter","Squamous cell carcinoma (n = 229)","Adenocarcinoma (n = 150)","p-value"],["- Surgical complications (%)","","",""],["- General complications (%)","37.9","33.7","0.447"],["-Total complications (%)","40.7","28.0","0.012*"],["- Length of ICU – stay (d)","68.1","57.6","0.039*"],["- 30-day-moratlity (... actual: {"rows":[["parameter","Squamous cell carcinoma (n = 229)","Adenocarcinoma (n = 150)","p-value"],["- Surgical complications (%)","","",""],["- General complications (%)","37.9","33.7","0.447"],["- Total complications (%)","40.7","28.0","0.012*"],["- Length of ICU – stay (d)","68.1","57.6","0.039*"],["- 30-day-mortality ... - v3-008 expected: {"rows":[["Drug","Drug carrier","Targeting moiety","Inhibition of proliferation (%)"],["None","None","None","0.0 ± 8.2"],["FUdR","None","Anti-CD134","8.9 ± 8.7"],["FUdR-dP","Liposomes","None","6.3 ± 0.7"],["None","Liposomes","Anti-CD134","0.6 ± 1.1"],["FUdR-dP","Liposomes","Anti-CD134","31.0 ± 2.3*"]]} actual: {"rows":[["Drug","Drug carrier","Targeting moiety","Inhibition of proliferation (%)"],["None","None","None","0.0 ± 8.2"],["FUdR","None","Anti-CD134","8.9 ± 8.7"],["FUdR-dP","Liposomes","None","6.3 ± 0.7"],["FUdR-dP","None","Anti-CD134","0.6 ± 1.1"],["FUdR-dP","Liposomes","Anti-CD134","31.0 ± 2.3*"]]} - v3-009 expected: {"rows":[["Relation","Transitive","Symmetric","Reflexive","Antisymmetric"],["is_a","+","-","+","+"],["part_of","+","-","+","+"],["located_in","+","-","+","-"],["contained_in","-","-","-","-"],["adjacent_to","-","-","-","-"],["transformation_of","+","-","-","-"],["derives_ from","+","-","-","-"],["preceded_by","+","-","... actual: {"rows":[["Relation","Transitive","Symmetric","Reflexive","Antisymmetric"],["is_a","+","-","+","+"],["part_of","+","-","+","+"],["located_in","+","-","+","-"],["contained_in","-","-","-","-"],["adjacent_to","-","+","-","-"],["transformation_of","+","-","-","-"],["derives_from","+","-","-","-"],["preceded_by","+","-","-... - v3-012 expected: {"rows":[["Allel","amino acid nr.","Sequence","Reference"],["HLA-A*0201","65–73","RMPEAAPPV","[115,155]"],["HLA-B*4601","99–107","SQKTYQGSY","[117]"],["HLA-A2","103–111","YQGSYGFRL","[120]"],["HLA-A24","125–134","TYSPALNKMF","[156]"],["HLA-A2","139–147","KTCPVQLWV","[120,157]"],["HLA-A2.1","149–157","STPPPGTRV","[84,12... actual: {"rows":[["Allel","amino acid nr.","Sequence","Reference"],["HLA-A*0201","65–73","RMPEAAPPV","[115,155]"],["HLA-B*4601","99–107","SQKTYQGSY","[115]"],["HLA-A2","103–111","YQGSYGFRL","[120]"],["HLA-A24","125–134","TYSPALNKMF","[156]"],["HLA-A24","139–147","KTCPVQLWV","[120,157]"],["HLA-A2.1","149–157","STPPPGTRV","[84,1... - v3-013 expected: {"rows":[["","VAS","","","VRS","",""],["Pain etiology group","PA (%)","RP (95% CI)","RC (95% CI)","PA (%)","RP (95% CI)","RC (95% CI)"],["All, n = 81","20","0.01 (-0.03 to 0.04)","-0.02 (-0.08 to 0.05)","94","0.03 (-0.01 to 0.06)","0.04 (-0.01 to 0.08)"],["Chron/idiop, n = 30","20","-0.05 (-0.12 to 0.01)","-0.07 (-0.18... actual: {"rows":[["Pain etiology group","VAS","","","VRS","",""],["","PA (%)","RP (95% CI)","RC (95% CI)","PA (%)","RP (95% CI)","RC (95% CI)"],["All, n = 81","20","0.01 (-0.03 to 0.04)","-0.02 (-0.08 to 0.05)","94","0.03 (-0.01 to 0.06)","0.04 (-0.01 to 0.08)"],["Chron/idiop, n = 30","20","-0.05 (-0.12 to 0.02)","-0.07 (-0.18... - v3-016 expected: {"rows":[["Study","No.","Mean dose μg/kg/day (range)","Mean T1/2 (days)","Mean elimination rate constant (b)"],["Cox et al. 1989","55","(1.2–79.6)a","47.2","0.0147"],["Miettinen et al. 1971","6","0.3b","49.9","0.0142"],["Sherlock et al. 1984","20","1.6 (0.5–3.6)","50.2","0.0140"],["Kershaw et al. 1980","4","20.0 (18.1–... actual: {"rows":[["Study","No.","Mean dose µg/kg/day (range)","Mean T1/2 (days)","Mean elimination rate constant (β)"],["Cox et al. 1989","55","(1.2–79.8)a","47.2","0.0147"],["Miettinen et al. 1971","6","0.3b","49.9","0.0140"],["Sherlock et al. 1984","20","1.6 (0.5–3.6)","50.2","0.0140"],["Kershaw et al. 1980","4","20.0 (18.1–... - v3-017 expected: {"rows":[["Parameter","Current centrala tendency estimate","Pregnancy specific?","Third-trimester specific?","EPA central tendency estimate","Pregnancy specific?","Third-trimester specific?"],["R","1.7","Yes","Yes","1.0 (implicit)","No","No"],["b","0.0147 day−1 (47 days)","Yes","No","0.014 day−1 (50 days)","No","No"],[... actual: {"rows":[["Parameter","Current centrala tendency estimate","Pregnancy specific?","Third-trimester specific?","EPA central tendency estimate","Pregnancy specific?","Third-trimester specific?"],["R","1.7","Yes","Yes","1.0 (implicit)","No","No"],["b","0.0147 day−1 (47 days)","Yes","Yes","0.014 day−1 (50 days)","No","No"],... - v3-018 expected: {"rows":[["Reference","Randomization","Sample size determination","Drop outs","Intention to treat analysis","Blinding","Allocation concealment"],["Kitchell 1963 [17]","Not described (nor stated)","Not described","33% of the treatment group upon cross-over","No","Allegedly participants blinded, but impossible to adequat... actual: {"rows":[["Reference","Randomization","Sample size determination","Drop outs","Intention to treat analysis","Blinding","Allocation concealment"],["Kitchell 1963 [17]","Not described (nor stated)","Not described","33% of the treatment group upon cross-over","No","Allegedly blinded, but impossible to adequately assess","... == IMAGE / test V5 UI screenshot / Claude Fable 5.1, as it ships (agent harness, no API) == (1 recorded, score 95.0, bar >= 80) - v5-015 expected: {"answers":["on","enabled"]} actual: {"answer":"Checked (enabled)"} == IMAGE / test V7 No fabrication from an image / Claude Fable 5.1, as it ships (agent harness, no API) == (2 recorded, score 1, bar <= 1) - v7-005 expected: {"present":true,"object":"skis"} actual: {"answer":null} - v7-023 expected: {"present":false,"object":"bus"} actual: {"answer":"yellow"}