@conference {1506983,
	title = {Is There a Chink in Your ARMOR? Towards Robust Evaluations for Deployed Security Systems },
	booktitle = {IJCAI 2009 Workshop on Quantitative Risk Analysis for Security Applications. QRASA-2009},
	year = {2009},
	abstract = {A growing number of security applications, designed to reduce risk from adversaries{\textquoteright} actions, are
being developed and deployed. However, there are
many challenges when attempting to evaluate such
systems, both in the lab and in the real world. Traditional evaluations used by computer scientists,
such as runtime analysis and optimality proofs,
may be largely irrelevant. The primary contribution of this paper is to provide a preliminary framework which can guide the evaluation of such systems and to apply the framework to the evaluation
of ARMOR (a system deployed at LAX since August 2007). This framework helps determine what
experiments could, and should, be run in order to
measure a system{\textquoteright}s overall utility. A secondary
contribution of this paper is to help familiarize our
community with some of the difficulties inherent in
evaluating deployed applications, focusing on those
in security domains.},
	author = {Matthew E. Taylor and Chris Kiekintveld and Craig Western and Tambe, Milind}
}
