results.csv

day,model,result,total_time
1,gpt-4o,"2264607
19457120
",0.0517818927764892
1,claude-3-5-sonnet-20241022,"2264607
19457120
",0.0311038494110107
1,jerpint,"2264607
19457120
",0.072559118270874
1,gemini-1.5-pro,"2264607
19457120
",0.0367422103881835
2,gpt-4o,"282
349
",0.0345420837402343
2,gemini-1.5-pro,"282
349
",0.0362250804901123
2,claude-3-5-sonnet-20241022,"282
349
",0.0348310470581054
2,jerpint,"282
349
",0.0335607528686523
3,jerpint,"173529487
99532691
",0.0314149856567382
3,gpt-4o,"173529487
99532691
",0.0277769565582275
3,gemini-1.5-pro,"173529487
173529487
",0.0306653976440429
3,claude-3-5-sonnet-20241022,"173529487
99532691
",0.0305140018463134
4,gpt-4o,"2493
122
",0.1320409774780273
4,gemini-1.5-pro,"3872
0
",0.1471152305603027
4,claude-3-5-sonnet-20241022,"2493
1890
",0.2896838188171386
4,jerpint,"2493
1890
",0.1274900436401367
5,gpt-4o,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day05/solution_gpt-4o.py"", line 77, in <module>
    main()
  File ""/Users/jeremypinto/advent24-llm/day05/solution_gpt-4o.py"", line 62, in main
    rules, updates = parse_input(file)
  File ""/Users/jeremypinto/advent24-llm/day05/solution_gpt-4o.py"", line 16, in parse_input
    updates.append(list(map(int, line.split(','))))
ValueError: invalid literal for int() with base 10: ''
",0.0285351276397705
5,gemini-1.5-pro,"5391
6142
",0.0330429077148437
5,claude-3-5-sonnet-20241022,"5391
6142
",0.0353639125823974
5,jerpint,"5391
6142
",2.074583053588867
6,jerpint,"4789
1304
",24.77995491027832
6,gpt-4o,"4789
1304
",67.0117359161377
6,claude-3-5-sonnet-20241022,Error: Timeout,300.0090808868408
6,gemini-1.5-pro,Error: Timeout,300.00918889045715
7,claude-3-5-sonnet-20241022,"5540634308362
472290821152397
",20.063307762145996
7,gemini-1.5-pro,"5540634308362
472290821152397
",3.241072177886963
7,jerpint,"5540634308362
472290821152397
",159.69412803649902
7,gpt-4o,"472290821152397
",19.688328981399536
8,jerpint,"273
1017
",0.0589988231658935
8,gpt-4o,"5070
1510
",0.0319371223449707
8,claude-3-5-sonnet-20241022,"0
1017
",0.3565587997436523
8,gemini-1.5-pro,Error: Timeout,300.0092422962189
9,claude-3-5-sonnet-20241022,"6421128769094
6448157430324
",192.63301491737369
9,gemini-1.5-pro,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day09/solution_gemini-1.5-pro.py"", line 78, in <module>
    result1 = solve_part1(disk_map)
  File ""/Users/jeremypinto/advent24-llm/day09/solution_gemini-1.5-pro.py"", line 9, in solve_part1
    free_size = int(disk_map[i+1])
IndexError: string index out of range
",0.0297279357910156
9,gpt-4o,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day09/solution_gpt-4o.py"", line 80, in <module>
    main()
  File ""/Users/jeremypinto/advent24-llm/day09/solution_gpt-4o.py"", line 69, in main
    disk = parse_disk_map(disk_map)
  File ""/Users/jeremypinto/advent24-llm/day09/solution_gpt-4o.py"", line 6, in parse_disk_map
    free_space_length = int(disk_map[i+1])
IndexError: string index out of range
",0.0369679927825927
9,jerpint,"6421128769094
6448168620520
",38.908868074417114
10,gemini-1.5-pro,"760
1764
",0.0449333190917968
10,gpt-4o,"760
1764
",0.0459368228912353
10,jerpint,"760
1764
",0.0722007751464843
10,claude-3-5-sonnet-20241022,"760
1764
",0.041105031967163
11,jerpint,"220722
261952051690787
",0.2919449806213379
11,claude-3-5-sonnet-20241022,Error: Timeout,300.5751881599426
11,gpt-4o,Error: Timeout,300.43601632118225
11,gemini-1.5-pro,Error: Timeout,300.45362615585327
12,gpt-4o,"1442713
1442713
",0.1200618743896484
12,gemini-1.5-pro,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day12/solution_gemini-1.5-pro.py"", line 86, in <module>
    solve()
  File ""/Users/jeremypinto/advent24-llm/day12/solution_gemini-1.5-pro.py"", line 68, in solve
    for rr, cc in bfs(r,c, char)[2]:
IndexError: tuple index out of range
",0.0341930389404296
12,claude-3-5-sonnet-20241022,"1489582
1489582
",0.1392021179199218
12,jerpint,"1489582
",0.186622142791748
13,gemini-1.5-pro,"0
0
",0.0350427627563476
13,gpt-4o,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day13/solution_gpt-4o.py"", line 33, in <module>
    machines = parse_input(file)
  File ""/Users/jeremypinto/advent24-llm/day13/solution_gpt-4o.py"", line 6, in parse_input
    a_x, a_y = map(int, lines[i].strip().split()[2::2])
ValueError: invalid literal for int() with base 10: 'X+19,'
",0.027897834777832
13,claude-3-5-sonnet-20241022,Error: Timeout,300.00735425949097
13,jerpint,"28262
101406661266314
",0.0829849243164062
14,jerpint,"210587128
7286
",0.0874340534210205
14,claude-3-5-sonnet-20241022,"197535240
353
",4.457779884338379
14,gemini-1.5-pro,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day14/solution_gemini-1.5-pro.py"", line 48, in <module>
    robots = parse_input(file)
  File ""/Users/jeremypinto/advent24-llm/day14/solution_gemini-1.5-pro.py"", line 9, in parse_input
    ((int(p_str[0]), int(p_str[1])), (int(v_str[0]), int(v_str[1])))
ValueError: invalid literal for int() with base 10: ''
",0.024846076965332
14,gpt-4o,"210587128
",0.0266840457916259
15,jerpint,"1360570
",0.146615982055664
15,claude-3-5-sonnet-20241022,"1412022
1424862
",0.9226350784301758
15,gemini-1.5-pro,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day15/solution_gemini-1.5-pro.py"", line 104, in <module>
    solve()
  File ""/Users/jeremypinto/advent24-llm/day15/solution_gemini-1.5-pro.py"", line 46, in solve
    if grid[r][c] == 'O':
IndexError: list index out of range
",0.0579340457916259
15,gpt-4o,"1354856
1369212
",0.0375778675079345
16,jerpint,"74392
",29.106717824935913
16,claude-3-5-sonnet-20241022,"74392
0
",0.0764460563659668
16,gemini-1.5-pro,"74392
2
",0.1279311180114746
16,gpt-4o,"1
",0.0732479095458984
17,jerpint,"7,3,5,7,5,7,4,3,0
",0.0280790328979492
17,gpt-4o,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day17/solution_gpt-4o.py"", line 65, in <module>
    print(part1(file))
  File ""/Users/jeremypinto/advent24-llm/day17/solution_gpt-4o.py"", line 51, in part1
    initial_values, program = parse_input(file)
  File ""/Users/jeremypinto/advent24-llm/day17/solution_gpt-4o.py"", line 5, in parse_input
    program = list(map(int, lines[3].split(': ')[1].split(',')))
IndexError: list index out of range
",0.0248689651489257
17,gemini-1.5-pro,"7,7,7,7,7,7,7,7,7
",16.903484106063843
17,claude-3-5-sonnet-20241022,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day17/solution_claude-3-5-sonnet-20241022.py"", line 87, in <module>
    main()
  File ""/Users/jeremypinto/advent24-llm/day17/solution_claude-3-5-sonnet-20241022.py"", line 78, in main
    registers, program = parse_input(""./input.txt"")
  File ""/Users/jeremypinto/advent24-llm/day17/solution_claude-3-5-sonnet-20241022.py"", line 12, in parse_input
    program = [int(x) for x in lines[-1].split(',')]
  File ""/Users/jeremypinto/advent24-llm/day17/solution_claude-3-5-sonnet-20241022.py"", line 12, in <listcomp>
    program = [int(x) for x in lines[-1].split(',')]
ValueError: invalid literal for int() with base 10: 'Program: 2'
",0.0285909175872802
18,jerpint,"330
10,38
",1.448005199432373
18,gpt-4o,"330
10,38
",9.956310033798218
18,gemini-1.5-pro,"330
10,38
",25.999710083007812
18,claude-3-5-sonnet-20241022,"330
10,38
",13.420971155166626
19,claude-3-5-sonnet-20241022,"290
712058625427487
",2.365529775619507
19,gpt-4o,"712058625427487
",0.9564287662506104
19,jerpint,"290
",0.10815191268920898
19,gemini-1.5-pro,Error: Timeout,300.0093183517456
20,gpt-4o,"20
",0.6666040420532227
20,jerpint,"(105, 35)
1375
",6.334330081939697
20,gemini-1.5-pro,Error: Timeout,300.0078308582306
20,claude-3-5-sonnet-20241022,"1
5
",145.0637719631195
21,jerpint,"157908
",0.0293052196502685
21,claude-3-5-sonnet-20241022,"157248
358176
",0.0261509418487548
21,gemini-1.5-pro,"0
0
",0.0287630558013916
21,gpt-4o,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day21/solution_gpt-4o.py"", line 54, in <module>
    main()
  File ""/Users/jeremypinto/advent24-llm/day21/solution_gpt-4o.py"", line 48, in main
    sequence = find_shortest_sequence(code, numeric_keypad, directions)
  File ""/Users/jeremypinto/advent24-llm/day21/solution_gpt-4o.py"", line 35, in find_shortest_sequence
    return precomputed_sequences[code]
KeyError: '341A'
",0.0282812118530273
22,claude-3-5-sonnet-20241022,Error: Timeout,300.0252139568329
22,gemini-1.5-pro,Error: Timeout,300.00811290740967
22,jerpint,"14082561342
1568
",54.82558584213257
22,gpt-4o,Error: Timeout,300.00778007507324
23,jerpint,"926
az,ed,hz,it,ld,nh,pc,td,ty,ux,wc,yg,zz
",2.992269992828369
23,claude-3-5-sonnet-20241022,Error: Timeout,300.0074918270111
23,gemini-1.5-pro,Error: Timeout,300.00904989242554
23,gpt-4o,Error: Timeout,300.00809812545776
24,gemini-1.5-pro,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day24/solution_gemini-1.5-pro.py"", line 37, in <module>
    z_values.append(str(evaluate_wire(f""z{i:02}"", circuit, cache)))
  File ""/Users/jeremypinto/advent24-llm/day24/solution_gemini-1.5-pro.py"", line 8, in evaluate_wire
    val1 = evaluate_wire(in1, circuit, cache)
  File ""/Users/jeremypinto/advent24-llm/day24/solution_gemini-1.5-pro.py"", line 4, in evaluate_wire
    if isinstance(circuit[wire], int):
KeyError: 'XOR'
",0.026757001876831
24,gpt-4o,"Error: Traceback (most recent call last):
  File ""/Users/jeremypinto/advent24-llm/day24/solution_gpt-4o.py"", line 54, in <module>
    wire_values = simulate_gates(wire_values, gate_operations)
  File ""/Users/jeremypinto/advent24-llm/day24/solution_gpt-4o.py"", line 21, in simulate_gates
    input1 = wire_values[parts[0]]
KeyError: ''
",0.0375468730926513
24,claude-3-5-sonnet-20241022,"26546678708189
bfw,ffh,frj,hwm,mjb,nrd,tgd,wpb
",0.0257329940795898
24,jerpint,"51657025112326
",0.0270252227783203
25,gpt-4o,"3732
",0.0514318943023681
25,claude-3-5-sonnet-20241022,"585
Merry Christmas!
",0.0423557758331298
25,jerpint,"3690
",0.0443761348724365
25,gemini-1.5-pro,"25250
There is no part 2 for this challenge.
",0.0472519397735595