1   /*
2    * Copyright (c) 2006, 2009 The Australian National University.
3    * All rights reserved. This program and the accompanying materials
4    * are made available under the terms of the Apache License v2.0.
5    * You may obtain the license at
6    * 
7    *    http://www.opensource.org/licenses/apache2.0.php
8    */
9   package org.dacapo.harness;
10  
11  import java.io.File;
12  import java.io.FileInputStream;
13  import java.io.FileNotFoundException;
14  import java.io.InputStream;
15  import java.lang.reflect.Constructor;
16  import java.lang.reflect.InvocationTargetException;
17  import java.net.URL;
18  import java.text.DecimalFormat;
19  import java.util.jar.Attributes;
20  import java.util.jar.Manifest;
21  import java.util.jar.JarFile;
22  import java.util.Locale;
23  
24  import org.dacapo.parser.Config;
25  
26  /**
27   * Main class for the Dacapo benchmark suite. Locates the configuration file for
28   * the specified benchmark, interprets command line arguments, and invokes the
29   * benchmark-specific harness class.
30   * 
31   * @date $Date: 2009-12-24 11:19:36 +1100 (Thu, 24 Dec 2009) $
32   * @id $Id: TestHarness.java 738 2009-12-24 00:19:36Z steveb-oss $
33   */
34  public class TestHarness {
35    public static final String PROP_BUILD_NICKNAME = "build.nickname";
36    public static final String PROP_BUILD_VERSION = "build.version";
37  
38    public static final String BUILD_NICKNAME = "Specification-Version";
39    public static final String BUILD_VERSION = "Implementation-Version";
40  
41    // these hold the build nick name and version strings respectively
42    private static String BuildNickName;
43    private static String BuildVersion;
44  
45    private final Config config;
46    private static CommandLineArgs commandLineArgs;
47  
48    public static final DecimalFormat two_dp = twoDecimalPlaces();
49  
50    public static String getBuildNickName() {
51      return BuildNickName;
52    }
53  
54    public static String getBuildVersion() {
55      return BuildVersion;
56    }
57  
58    private static URL getURL(String fn) {
59      ClassLoader cl = TestHarness.class.getClassLoader();
60      if (commandLineArgs.getVerbose())
61        System.out.println("TestHarness.getURL: returns " + cl.getResource(fn));
62      return cl.getResource(fn);
63    }
64  
65    public static boolean exists(File f) {
66      return exists(f.getPath());
67    }
68  
69    public static boolean exists(String fn) {
70      return getURL(fn) != null;
71    }
72  
73    /**
74     * Calculates coefficient of variation of a set of longs (standard deviation
75     * divided by mean).
76     * 
77     * @param times Array of input values
78     * @return Coefficient of variation
79     */
80    public static double coeff_of_var(long[] times) {
81      double n = times.length;
82      double sum = 0.0;
83      double sum2 = 0.0;
84  
85      for (int i = 0; i < times.length; i++) {
86        double x = times[i];
87        sum += x;
88        sum2 += x * x;
89      }
90  
91      double mean = sum / n;
92      double sigma = Math.sqrt(1.0 / n * sum2 - mean * mean);
93  
94      return sigma / mean;
95    }
96  
97    public static void main(String[] args) {
98      // force the locale so that we don't have any character set issues
99      // when generating output for the digests.
100     Locale.setDefault(new Locale("en", "AU"));
101 
102     /* All benchmarks run headless */
103     System.setProperty("java.awt.headless", "true");
104 
105     try {
106       commandLineArgs = new CommandLineArgs(args);
107 
108       File scratch = new File(commandLineArgs.getScratchDir());
109       makeCleanScratch(scratch);
110 
111       // this is not right
112       Benchmark.setCommandLineOptions(commandLineArgs);
113       try {
114         Config.setThreadCountOverride(Integer.parseInt(commandLineArgs.getThreadCount()));
115       } catch (RuntimeException re) {
116       }
117 
118       // now get the benchmark names and run them
119       for (String bm : commandLineArgs.benchmarks()) {
120         // check if it is a benchmark name
121         // name of file containing configurations
122         InputStream ins = null;
123         if (commandLineArgs.getCnfOverride() == null) {
124           String cnf = "cnf/" + bm + ".cnf";
125           ins = TestHarness.class.getClassLoader().getResourceAsStream(cnf);
126           if (ins == null) {
127             System.err.println("Unknown benchmark: " + bm);
128             System.exit(20);
129           }
130         } else {
131           String cnf = commandLineArgs.getCnfOverride();
132           try {
133             ins = new FileInputStream(cnf);
134           } catch (FileNotFoundException e) {
135             System.err.println("Count not find cnf file: '" + cnf + "'");
136             System.exit(20);
137           }
138         }
139   
140         TestHarness harness = new TestHarness(ins);
141 
142         String size = commandLineArgs.getSize();
143 
144         int factor = 0;
145         int limit = harness.config.getThreadLimit(size);
146 
147         try {
148           factor = Integer.parseInt(commandLineArgs.getThreadFactor());
149           if (0 < factor && harness.config.getThreadModel() == Config.ThreadModel.PER_CPU)
150             harness.config.setThreadFactor(size, factor);
151         } catch (RuntimeException re) {
152         }
153 
154         if (!harness.isValidSize(size)) {
155           System.err.println("No configuration size, " + size + ", for benchmark " + bm + ".");
156         } else if (factor != 0 && harness.config.getThreadModel() != Config.ThreadModel.PER_CPU) {
157           System.err.println("Can only set the thread factor for per_cpu configurable benchmarks");
158         } else if (!harness.isValidThreadCount(size) && (harness.config.getThreadCountOverride() > 0 || factor > 0)) {
159           System.err.println("The specified number of threads (" + harness.config.getThreadCount(size) + ") is outside the range [1,"
160               + (limit == 0 ? "unlimited" : "" + limit) + "]");
161         } else if (commandLineArgs.getInformation()) {
162           harness.bmInfo(size);
163         } else {
164           if (!harness.isValidThreadCount(size)) {
165             System.err.println("The derived number of threads (" + harness.config.getThreadCount(size) + ") is outside the range [1,"
166                 + (limit == 0 ? "unlimited" : "" + limit) + "]; rescaling to match thread limit.");
167             harness.config.setThreadCountOverride(harness.config.getThreadLimit(size));
168           }
169 
170           harness.dump(commandLineArgs.getVerbose());
171           runBenchmark(scratch, bm, harness);
172         }
173       }
174     } catch (Exception e) {
175       System.err.println(e);
176       e.printStackTrace();
177       System.exit(-1);
178     }
179   }
180 
181   public static void makeCleanScratch(File scratch) {
182     rmdir(scratch);
183     scratch.mkdir();
184   }
185 
186   private boolean isValidSize(String size) {
187     return size != null && config.getSizes().contains(size);
188   }
189 
190   private boolean isValidThreadCount(String size) {
191     return config.getThreadLimit(size) == 0 || config.getThreadCount(size) <= config.getThreadLimit(size);
192   }
193 
194   /**
195    * @param scratch
196    * @param bm
197    * @param harness
198    * @param c
199    * @throws NoSuchMethodException
200    * @throws InstantiationException
201    * @throws IllegalAccessException
202    * @throws InvocationTargetException
203    * @throws Exception
204    */
205   private static void runBenchmark(File scratch, String bm, TestHarness harness) throws NoSuchMethodException, InstantiationException, IllegalAccessException,
206       InvocationTargetException, Exception {
207     harness.config.printThreadModel(System.out, commandLineArgs.getSize(), commandLineArgs.getVerbose());
208 
209     Constructor<?> cons = harness.findClass().getConstructor(new Class[] { Config.class, File.class });
210 
211     Benchmark b = (Benchmark) cons.newInstance(new Object[] { harness.config, scratch });
212 
213     boolean valid = true;
214     Callback callback = commandLineArgs.getCallback();
215     callback.init(harness.config);
216 
217     do {
218       valid = b.run(callback, commandLineArgs.getSize()) && valid;
219     } while (callback.runAgain());
220     b.cleanup();
221 
222     if (!valid) {
223       System.err.println("Validation FAILED for " + bm + " " + commandLineArgs.getSize());
224       if (!commandLineArgs.getIgnoreValidation())
225         System.exit(-2);
226     }
227   }
228 
229   /**
230    * @return A Decimal Format object
231    */
232   private static DecimalFormat twoDecimalPlaces() {
233     DecimalFormat two_dp;
234     two_dp = new DecimalFormat();
235     two_dp.setMaximumFractionDigits(2);
236     two_dp.setMinimumFractionDigits(2);
237     two_dp.setGroupingUsed(true);
238     return two_dp;
239   }
240 
241   private static void rmdir(File dir) {
242     String[] files = dir.list();
243     if (files != null) {
244       for (int f = 0; f < files.length; f++) {
245         File file = new File(dir, files[f]);
246         if (file.isDirectory())
247           rmdir(file);
248         if (!file.delete())
249           System.err.println("Could not delete " + files[f]);
250       }
251     }
252   }
253 
254   public static int TEST(int i) {
255     System.err.println("In TEST");
256     System.err.println(i);
257     return 2 * i;
258   }
259 
260   private void bmInfo(String size) {
261     config.describe(System.err, size);
262   }
263 
264   private void dump(boolean verbose) {
265     if (verbose) {
266       System.err.println("Class name: " + config.className);
267 
268       System.err.println("Configurations:");
269       config.describe(System.err, commandLineArgs.getSize());
270     }
271   }
272 
273   private TestHarness(InputStream ins) {
274      config = Config.parse(ins);
275      if (config == null)
276         System.exit(-1);
277   }
278 
279   private Class<?> findClass() {
280     try {
281       return Class.forName(config.className);
282     } catch (ClassNotFoundException e) {
283       System.err.println(e);
284       e.printStackTrace();
285       System.exit(-1);
286       return null; // not reached
287     }
288   }
289 
290   {
291       BuildNickName = "Unknown";
292       BuildVersion = "unknown";
293 
294     try {
295       JarFile jarFile = new JarFile(new File(TestHarness.class.getProtectionDomain().getCodeSource().getLocation().getFile()));
296 
297       Manifest manifest = jarFile.getManifest();
298       Attributes attributes = manifest.getMainAttributes();
299 
300       String nickname = attributes.get(new Attributes.Name(BUILD_NICKNAME)).toString();
301       String version = attributes.get(new Attributes.Name(BUILD_VERSION)).toString();
302 
303       BuildNickName = nickname;
304       BuildVersion = version;
305     } catch (Exception e) {
306       BuildNickName = "Unknown";
307       BuildVersion = "unknown";
308     }
309   }
310 }
311