@conference {1502680,
	title = {Beware the Soothsayer: From Attack Prediction Accuracy to Predictive Reliability in Security Games },
	booktitle = {Conference on Decision and Game Theory for Security},
	year = {2015},
	abstract = {. Interdicting the flow of illegal goods (such as drugs and ivory) is a
major security concern for many countries. The massive scale of these networks,
however, forces defenders to make judicious use of their limited resources. While
existing solutions model this problem as a Network Security Game (NSG), they
do not consider humans{\textquoteright} bounded rationality. Previous human behavior modeling
works in Security Games, however, make use of large training datasets that are
unrealistic in real-world situations; the ability to effectively test many models is
constrained by the time-consuming and complex nature of field deployments. In
addition, there is an implicit assumption in these works that a model{\textquoteright}s prediction
accuracy strongly correlates with the performance of its corresponding defender
strategy (referred to as predictive reliability). If the assumption of predictive reliability does not hold, then this could lead to substantial losses for the defender. In
the following paper, we (1) first demonstrate that predictive reliability is indeed
strong for previous Stackelberg Security Game experiments. We also run our own
set of human subject experiments in such a way that models are restricted to
learning on dataset sizes representative of real-world constraints. In the analysis
on that data, we demonstrate that (2) predictive reliability is extremely weak for
NSGs. Following that discovery, however, we identify (3) key factors that influence predictive reliability results: the training set{\textquoteright}s exposed attack surface and
graph structure.},
	author = {Ford, Benjamin and Nguyen, Thanh and Tambe, Milind and Sintov, Nicole and Delle Fave, Francesco}
}
