#
# ATP evaluation
# ==============
#
#
# To run e.g. one of the evaluations of the "bushy" dataset,
# first install the dataset via:
#
#     make bushy
#
# Then, compile the CoP binaries required for the experiments.
# This step should be done before running the evaluation,
# because after the evaluation, the binaries are removed by `make`
# if they did not exist before, provoking a rebuild of all the
# experiments in some cases.
#
#     make cop-170209/lazycop.native cop-170209/montecop.native
#
# Finally, run an evaluation, such as
#
#     make out/bushy/10s/defcnf/montecop-170209-mlrew
#
# If the experiment command mentions data files (e.g. ldata or cdata),
# build these first, e.g. such as
#
#     make out/bushy/60s/defcnf/lazycop-170209-md10.ldata
#
# To evaluate the outcome of the experiments (e.g. the solved problems), run
#
#     make solved/bushy/10s/defcnf/montecop-170209-mlrew
#
# Because some of the Makefiles use macros, please use a version of
# GNU make that supports them. Version 3.81 does not. We use version 4.1.
#

include datasets.mk
include cop.mk

include bushy-1s.mk
include bushy-10s.mk
include bushy-20s.mk
include bushy-60s.mk
include tptp-1s.mk
include mzrtest-10s.mk
include mzrtrain-300s.mk
include paramils.mk


# collect solved files for a given prover
solved/%: out/%
	@mkdir -p "`dirname $@`"
	grep -rl $< --exclude "*.trace" -e "Theorem" | xargs -L1 basename | sort > "$@"

tables/%:
	@mkdir -p "`dirname $@`"
	wc -l solved/$** | sed \$$d | \
	  awk '{print $$2 " " $$1}' | sed 's|.*solved/$*||' | sort -n > $@

discr/%: out/%
	@mkdir -p "`dirname $@`"
	sed -n 's|.*UCTDiscr: \(.*\)/\(.*\)|\1 \2|p' $</*.p | grep -v nan | \
          awk '{s1 += $$1; s2 += $$2; n++} END {print s1/n " " s2/n}'
