Benchmarking speedups

.gitignore

 bin
 out
-progs/*.txt
+progs/*-actual.txt
+progs/*-expected.txt
+progs/*-time.csv

Makefile

@@ -2,22 +2,30 @@ CC = clang-with-asan
 CFLAGS = -Iinclude -Wall -Wextra -fno-sanitize=integer
 ASM = clang
 
-TESTS_1 =            $(sort $(wildcard progs/stage1-*.bas))
-TESTS_2 = $(TESTS_1) $(sort $(wildcard progs/stage2-*.bas))
-TESTS_3 = $(TESTS_2) $(sort $(wildcard progs/stage3-*.bas))
-TESTS_4 = $(TESTS_3) $(sort $(wildcard progs/stage4-*.bas))
-TESTS_5 = $(TESTS_4) $(sort $(wildcard progs/stage5-*.bas))
-TESTS_6 = $(TESTS_5) $(sort $(wildcard progs/stage6-*.bas))
-TESTS_7 = $(TESTS_6) $(sort $(wildcard progs/stage7-*.bas))
-
-test: test7
-test1: $(TESTS_1:progs/%.bas=%-result)
-test2: $(TESTS_2:progs/%.bas=%-result)
-test3: $(TESTS_3:progs/%.bas=%-result)
-test4: $(TESTS_4:progs/%.bas=%-result)
-test5: $(TESTS_5:progs/%.bas=%-result)
-test6: $(TESTS_6:progs/%.bas=%-result)
-test7: $(TESTS_7:progs/%.bas=%-result)
+COMPILE_TESTS_1 =                    $(sort $(wildcard progs/stage1-*.bas))
+COMPILE_TESTS_2 = $(COMPILE_TESTS_1) $(sort $(wildcard progs/stage2-*.bas))
+COMPILE_TESTS_3 = $(COMPILE_TESTS_2) $(sort $(wildcard progs/stage3-*.bas))
+COMPILE_TESTS_4 = $(COMPILE_TESTS_3) $(sort $(wildcard progs/stage4-*.bas))
+COMPILE_TESTS_5 = $(COMPILE_TESTS_4) $(sort $(wildcard progs/stage5-*.bas))
+COMPILE_TESTS_6 = $(COMPILE_TESTS_5) $(sort $(wildcard progs/stage6-*.bas))
+COMPILE_TESTS_7 = $(COMPILE_TESTS_6) $(sort $(wildcard progs/stage7-*.bas))
+
+OPT_TESTS_1 = stage7-unhash
+OPT_TESTS_2 = stage7-loops-of-ops
+
+all: compile opt1 opt2
+
+compile: compile7
+compile1: $(COMPILE_TESTS_1:progs/%.bas=%-result)
+compile2: $(COMPILE_TESTS_2:progs/%.bas=%-result)
+compile3: $(COMPILE_TESTS_3:progs/%.bas=%-result)
+compile4: $(COMPILE_TESTS_4:progs/%.bas=%-result)
+compile5: $(COMPILE_TESTS_5:progs/%.bas=%-result)
+compile6: $(COMPILE_TESTS_6:progs/%.bas=%-result)
+compile7: $(COMPILE_TESTS_7:progs/%.bas=%-result)
+
+opt1: $(OPT_TESTS_1:=-bench)
+opt2: $(OPT_TESTS_2:=-bench)
 
 out/%.o: src/%.c
 	$(CC) $(CFLAGS) -c $^ -o $@
@@ -51,15 +59,11 @@ progs/%-actual.txt: bin/%
 progs/%-time.csv: bin/time-%
 	$^ > $@
 
-progs/times.csv: $(TESTS_7:.bas=-time.csv)
-	echo test_name,mean_log_duration,variance_log_duration > $@
-	cat $^ >> $@
-
-times: compare_times.py reference-times.csv progs/times.csv
+%-bench: compare_times.py reference-times.csv progs/%-time.csv progs/%-speedup.txt
 	./$^
 
 clean:
-	rm -f out/* bin/* progs/*.txt progs/*.csv
+	rm -f out/* bin/* progs/*-expected.txt progs/*-actual.txt progs/*-time.csv
 
 .PRECIOUS: out/%.o out/%.s bin/% bin/time-% \
 	progs/%-expected.txt progs/%-actual.txt progs/%-time.csv

compare_times.py

@@ -16,19 +16,24 @@ def read_times(filename):
     return times
 
 if __name__ == '__main__':
-    (_, old_file, new_file) = sys.argv
-    old_times = read_times(old_file)
+    (_, reference_file, new_file, speedup_file) = sys.argv
+    reference_times = read_times(reference_file)
     new_times = read_times(new_file)
-    for test_name, time in old_times.items():
-        new_time = new_times[test_name]
-        speedup_string = test_name + ': '
+    with open(speedup_file) as f:
+        expected_speedup = float(f.read().strip())
 
-        mean_speedup = math.expm1(time['mean_log'] - new_time['mean_log'])
-        speedup_string += str(abs(mean_speedup * 100)) + '%'
-        speedup_string += ' faster' if mean_speedup > 0 else ' slower'
+    ((test_name, new_time),) = new_times.items()
+    time = reference_times[test_name]
+    mean_speedup = math.expm1(time['mean_log'] - new_time['mean_log'])
+    speedup_percent = abs(mean_speedup * 100)
+    speedup_direction = 'faster' if mean_speedup > 0 else 'slower'
+    variance_log_sum = time['variance_log'] + new_time['variance_log']
+    variance_percent = math.expm1(math.sqrt(variance_log_sum)) * 100
+    print(f'{test_name} ran {speedup_percent}% {speedup_direction} (+/- {variance_percent}%)')
 
-        variance_log_sum = time['variance_log'] + new_time['variance_log']
-        variance_speedup = math.expm1(math.sqrt(variance_log_sum))
-        speedup_string += ' (+/- ' + str(variance_speedup * 100) + ' %)'
+    print(f'Expected speedup: {expected_speedup * 100}% faster')
+
+    if mean_speedup < expected_speedup:
+        print('Make it faster!')
+        sys.exit(1)
 
-        print(speedup_string)

progs/stage7-loops-of-ops-speedup.txt

+1

progs/stage7-unhash-speedup.txt

+0.3

reference-times.csv

+test_name,mean_log_duration,variance_log_duration
+stage1-1337,-15.414441,4.630224e-09
+stage1-42,-15.423754,3.730815e-09
+stage1-print-multiple,-14.880734,5.551399e-09
+stage2-1-plus-1,-15.417299,3.838244e-09
+stage2-1-plus-2-plus3,-15.399964,4.883277e-09
+stage2-1-plus-5,-15.418377,4.506159e-09
+stage2-5-plus-1,-15.416418,3.775031e-09
+stage2-big-stack,-15.048829,4.927516e-09
+stage2-overflow,-15.414238,4.393723e-09
+stage3-bad-order,-15.415961,4.549370e-09
+stage3-lots-of-ops,-12.756345,4.280501e-08
+stage3-overflow,-15.272850,4.365156e-09
+stage4-basic-division,-14.941708,5.003680e-09
+stage4-lots-of-ops,-11.212322,1.852049e-07
+stage5-big-locals,-15.223882,4.708205e-09
+stage5-local-test,-15.373016,3.674967e-09
+stage5-local-test2,-15.343883,4.535228e-09
+stage5-lots-of-ops,-10.598737,3.448534e-07
+stage6-comparisons,-15.313315,3.708140e-09
+stage7-bswap,-14.892616,5.035440e-09
+stage7-count-to-20,-15.085020,4.816644e-09
+stage7-count-together,-13.403312,2.162844e-08
+stage7-count-up-down,-14.693660,6.327085e-09
+stage7-digit-powers,-2.245703,2.760769e-04
+stage7-double-palindromes,-3.006717,1.672314e-04
+stage7-exponentiation,-10.989014,2.313973e-07
+stage7-fizz-buzz,-12.930649,4.318885e-08
+stage7-lcm,-8.051964,3.464892e-06
+stage7-loops-of-ops,-0.964310,1.200135e-04
+stage7-pascals-triangle,-11.415512,2.142535e-07
+stage7-pi-approx,-13.417496,2.351959e-08
+stage7-pi-exact,-0.860421,6.465658e-04
+stage7-primes,-6.798341,1.164016e-05
+stage7-riemann-sum,-5.495665,5.578510e-06
+stage7-sin,-8.071598,3.286681e-06
+stage7-sqrt,-13.943080,1.326310e-08
+stage7-square-digits,1.451112,8.851105e-06
+stage7-unhash,1.324347,1.066462e-08

runtime/timing.c

@@ -48,7 +48,11 @@ int main(int argc, char *argv[]) {
     double mean_log_duration = duration_log_sum / runs;
     double variance_log_duration =
         (duration_log_square_sum / runs - mean_log_duration * mean_log_duration) / runs;
-    printf("%s,%f,%e\n", test_name, mean_log_duration, variance_log_duration);
+    printf(
+        "test_name,mean_log_duration,variance_log_duration\n"
+        "%s,%f,%e\n",
+        test_name, mean_log_duration, variance_log_duration
+    );
     fprintf(stderr, "%s mean duration: %e seconds (+/- %e x)\n",
         test_name, exp(mean_log_duration), expm1(sqrt(variance_log_duration))
     );