001package org.w3.ldp.testsuite; 002 003import java.io.File; 004import java.net.URI; 005import java.net.URISyntaxException; 006import java.util.*; 007import java.util.regex.Pattern; 008 009import org.apache.commons.cli.BasicParser; 010import org.apache.commons.cli.CommandLine; 011import org.apache.commons.cli.CommandLineParser; 012import org.apache.commons.cli.HelpFormatter; 013import org.apache.commons.cli.Option; 014import org.apache.commons.cli.OptionBuilder; 015import org.apache.commons.cli.OptionGroup; 016import org.apache.commons.cli.Options; 017import org.apache.commons.cli.ParseException; 018import org.apache.commons.lang3.StringUtils; 019import org.apache.commons.lang3.exception.ExceptionUtils; 020import org.testng.IMethodInstance; 021import org.testng.IMethodInterceptor; 022import org.testng.ITestContext; 023import org.testng.TestNG; 024import org.testng.xml.XmlClass; 025import org.testng.xml.XmlSuite; 026import org.testng.xml.XmlTest; 027import org.w3.ldp.testsuite.reporter.LdpEarlReporter; 028import org.w3.ldp.testsuite.reporter.LdpHtmlReporter; 029import org.w3.ldp.testsuite.reporter.LdpTestListener; 030import org.w3.ldp.testsuite.test.LdpTest; 031import org.w3.ldp.testsuite.transformer.MethodEnabler; 032import org.w3.ldp.testsuite.util.OptionsHandler; 033 034import com.jayway.restassured.RestAssured; 035 036/** 037 * LDP Test Suite Command-Line Interface, a wrapper to {@link org.testng.TestNG} 038 * 039 * @author Sergio Fernández 040 * @author Steve Speicher 041 * @author Samuel Padgett 042 */ 043public class LdpTestSuite { 044 045 public static final String NAME = "LDP Test Suite"; 046 public static final String SPEC_URI = "http://www.w3.org/TR/ldp"; 047 public static final String OUTPUT_DIR = "report"; 048 049 static final String[] EARLDEPEDENTARGS = {"software", "developer", "language", "homepage", "assertor", "shortname"}; 050 051 private final TestNG testng; 052 053 private final List<XmlClass> classList; // for test types to add in 054 055 private final String reportTitle; 056 057 private String outputDir; 058 059 enum ContainerType { 060 BASIC, DIRECT, INDIRECT 061 } 062 063 /** 064 * Initialize the test suite with options as a map 065 * 066 * @param options map of options 067 */ 068 public LdpTestSuite(final Map<String, String> options) { 069 this(new OptionsHandler(options)); 070 } 071 072 /** 073 * Initialize the test suite with options as the row command-line input 074 * 075 * @param cmd command-line options 076 */ 077 public LdpTestSuite(final CommandLine cmd) { 078 this(new OptionsHandler(cmd)); 079 } 080 081 /** 082 * Initialize the test suite with options as the row command-line input 083 * 084 * @param cmd command-line options 085 * @param reportTitle report title 086 */ 087 public LdpTestSuite(final CommandLine cmd, String reportTitle) { 088 this(new OptionsHandler(cmd), reportTitle); 089 } 090 091 private LdpTestSuite(OptionsHandler optionsHandler) { 092 this(optionsHandler, null); 093 } 094 095 private LdpTestSuite(OptionsHandler optionsHandler, String reportTitle) { 096 // see: http://testng.org/doc/documentation-main.html#running-testng-programmatically 097 testng = new TestNG(); 098 this.reportTitle = reportTitle; 099 this.outputDir = OUTPUT_DIR; 100 this.classList = new ArrayList<>(); 101 this.setupSuite(optionsHandler); 102 } 103 104 public void checkUriScheme(String uri) throws URISyntaxException { 105 String scheme = new URI(uri).getScheme(); 106 if (!"http".equals(scheme) && !"https".equals(scheme)) { 107 throw new IllegalArgumentException("non-http uri"); 108 } 109 } 110 111 private void setupSuite(OptionsHandler options) { 112 113 testng.setDefaultSuiteName(NAME); 114 115 // create XmlSuite instance 116 XmlSuite testsuite = new XmlSuite(); 117 testsuite.setName(NAME); 118 119 // provide included/excluded groups 120 // get groups to include 121 final String[] includedGroups; 122 if(options.hasOption("includedGroups")) { 123 includedGroups = options.getOptionValues("includedGroups"); 124 for(String group : includedGroups){ 125 testsuite.addIncludedGroup(group); 126 } 127 } else{ 128 testsuite.addIncludedGroup(LdpTest.MUST); 129 testsuite.addIncludedGroup(LdpTest.SHOULD); 130 testsuite.addIncludedGroup(LdpTest.MAY); 131 } 132 // get groups to exclude 133 final String[] excludedGroups; 134 if(options.hasOption("excludedGroups")){ 135 excludedGroups = options.getOptionValues("excludedGroups"); 136 for(String group : excludedGroups){ 137 testsuite.addExcludedGroup(group); 138 } 139 } 140 141 // create XmlTest instance 142 XmlTest test = new XmlTest(testsuite); 143 test.setName("W3C Linked Data Platform Tests"); 144 145 // Add any parameters that you want to set to the Test. 146 // Test suite parameters 147 final Map<String, String> parameters = new HashMap<>(); 148 149 if (options.hasOption("output")) { 150 final String output = options.getOptionValue("output"); 151 outputDir = output + File.separator + LdpTestSuite.OUTPUT_DIR; 152 testng.setOutputDirectory(output + File.separator + TestNG.DEFAULT_OUTPUTDIR); 153 } 154 parameters.put("output", outputDir); 155 156 final String server; 157 if (options.hasOption("server")) { 158 server = options.getOptionValue("server"); 159 if (StringUtils.startsWith(server, "https:")) { // allow self-signed certificates for development servers 160 RestAssured.useRelaxedHTTPSValidation(); 161 } 162 try { 163 checkUriScheme(server); 164 } catch (Exception e) { 165 throw new IllegalArgumentException("ERROR: invalid server uri, " + e.getLocalizedMessage()); 166 } 167 } else { 168 throw new IllegalArgumentException("ERROR: missing server uri"); 169 } 170 171 // Listener injection from options 172 final String[] listeners; 173 if (options.hasOption("listeners")) { 174 listeners = options.getOptionValues("listeners"); 175 for (String listener : listeners) { 176 try { 177 Class<?> listenerCl = Class.forName(listener); 178 Object instance = listenerCl.newInstance(); 179 testng.addListener(instance); 180 } catch (ClassNotFoundException e) { 181 throw new IllegalArgumentException("ERROR: invalid listener class name, " + e.getLocalizedMessage()); 182 } catch (InstantiationException | IllegalAccessException e) { 183 throw new IllegalArgumentException("ERROR: problem while creating listener, " + e.getLocalizedMessage()); 184 } 185 } 186 } 187 188 // Add method enabler (Annotation Transformer) 189 testng.addListener(new MethodEnabler()); 190 191 testng.addListener(new LdpTestListener()); 192 LdpHtmlReporter reporter = new LdpHtmlReporter(); 193 if (StringUtils.isNotBlank(reportTitle)) { 194 reporter.setTitle(reportTitle); 195 } 196 reporter.setOutputDirectory(outputDir); 197 testng.addListener(reporter); 198 199 if (options.hasOption("earl")) { 200 LdpEarlReporter earlReport = new LdpEarlReporter(); 201 if (StringUtils.isNotBlank(reportTitle)) { 202 earlReport.setTitle(reportTitle); 203 } 204 earlReport.setOutputDirectory(outputDir); 205 testng.addListener(earlReport); 206 207 // required --earl args 208 for (String arg: EARLDEPEDENTARGS) { 209 if (options.hasOptionWithValue(arg)) 210 parameters.put(arg, options.getOptionValue(arg)); 211 else 212 printEarlUsage(arg); 213 } 214 215 // optional --earl args 216 if (options.hasOptionWithValue("mbox")) { 217 parameters.put("mbox", options.getOptionValue("mbox")); 218 } 219 220 } 221 222 if (options.hasOptionWithValue("cont-res")) { 223 final String containerAsResource = options.getOptionValue("cont-res"); 224 try { 225 checkUriScheme(containerAsResource); 226 } catch (Exception e) { 227 throw new IllegalArgumentException("ERROR: invalid containerAsResource uri, " + e.getLocalizedMessage()); 228 } 229 parameters.put("containerAsResource", containerAsResource); 230 } 231 232 if (options.hasOption("read-only-prop")) { 233 parameters.put("readOnlyProp", options.getOptionValue("read-only-prop")); 234 } 235 236 if (options.hasOption("relative-uri")) { 237 parameters.put("relativeUri", options.getOptionValue("relative-uri")); 238 } 239 240 if (options.hasOptionWithValue("auth")) { 241 final String auth = options.getOptionValue("auth"); 242 if (auth.contains(":")) { 243 String[] split = auth.split(":"); 244 if (split.length == 2 && StringUtils.isNotBlank(split[0]) && StringUtils.isNotBlank(split[1])) { 245 parameters.put("auth", auth); 246 } else { 247 throw new IllegalArgumentException("ERROR: invalid basic authentication credentials"); 248 } 249 } else { 250 throw new IllegalArgumentException("ERROR: invalid basic authentication credentials"); 251 } 252 } 253 254 final String postTtl; 255 if (options.hasOption("postTtl")) { 256 postTtl = options.getOptionValue("postTtl"); 257 parameters.put("postTtl", postTtl); 258 } 259 260 final String memberTtl; 261 if (options.hasOption("memberTtl")) { 262 memberTtl = options.getOptionValue("memberTtl"); 263 parameters.put("memberTtl", memberTtl); 264 } 265 266 final String memberResource; 267 if (options.hasOption("memberResource")) { 268 memberResource = options.getOptionValue("memberResource"); 269 parameters.put("memberResource", memberResource); 270 } 271 272 ContainerType type = getSelectedType(options); 273 switch (type) { 274 case BASIC: 275 classList.add(new XmlClass("org.w3.ldp.testsuite.test.BasicContainerTest")); 276 parameters.put("basicContainer", server); 277 break; 278 case DIRECT: 279 classList.add(new XmlClass("org.w3.ldp.testsuite.test.DirectContainerTest")); 280 parameters.put("directContainer", server); 281 break; 282 case INDIRECT: 283 classList.add(new XmlClass("org.w3.ldp.testsuite.test.IndirectContainerTest")); 284 parameters.put("indirectContainer", server); 285 break; 286 } 287 288 classList.add(new XmlClass("org.w3.ldp.testsuite.test.MemberResourceTest")); 289 testsuite.addIncludedGroup("ldpMember"); 290 291 if (options.hasOption("non-rdf")) { 292 classList.add(new XmlClass("org.w3.ldp.testsuite.test.NonRDFSourceTest")); 293 } 294 295 if (options.hasOption("httpLogging")) { 296 parameters.put("httpLogging", "true"); 297 } 298 299 if (options.hasOption("skipLogging")) { 300 parameters.put("skipLogging", "true"); 301 } 302 303 test.setXmlClasses(classList); 304 305 final List<XmlTest> tests = new ArrayList<>(); 306 tests.add(test); 307 308 testsuite.setParameters(parameters); 309 testsuite.setTests(tests); 310 311 final List<XmlSuite> suites = new ArrayList<>(); 312 suites.add(testsuite); 313 testng.setXmlSuites(suites); 314 315 if (options.hasOption("test")) { 316 final String[] testNamePatterns = options.getOptionValues("test"); 317 for (int i = 0; i < testNamePatterns.length; i++) { 318 // We support only * as a wildcard character to keep the command line simple. 319 // Convert the wildcard pattern into a regex to use internally. 320 testNamePatterns[i] = wildcardPatternToRegex(testNamePatterns[i]); 321 } 322 323 // Add a method intercepter to filter the list for matching tests. 324 testng.addListener(new IMethodInterceptor() { 325 @Override 326 public List<IMethodInstance> intercept(List<IMethodInstance> methods, ITestContext context) { 327 ArrayList<IMethodInstance> toRun = new ArrayList<>(); 328 for (IMethodInstance method : methods) { 329 for (String testNamePattern : testNamePatterns) { 330 if (method.getMethod().getMethodName().matches(testNamePattern)) { 331 toRun.add(method); 332 } 333 } 334 } 335 return toRun; 336 } 337 }); 338 } 339 } 340 341 private ContainerType getSelectedType(OptionsHandler options) { 342 if (options.hasOption("direct")) { 343 return ContainerType.DIRECT; 344 } else if (options.hasOption("indirect")) { 345 return ContainerType.INDIRECT; 346 } else { 347 return ContainerType.BASIC; 348 } 349 } 350 351 public void addTestClass(String klass) { 352 addTestClass(new XmlClass(klass)); 353 } 354 355 public void addTestClass(XmlClass klass) { 356 this.classList.add(klass); 357 } 358 359 public void addTestClasses(Collection<XmlClass> classes) { 360 this.classList.addAll(classes); 361 } 362 363 public String wildcardPatternToRegex(String wildcardPattern) { 364 // use lookarounds and zero-width matches to include the * delimeter in the result 365 String[] tokens = wildcardPattern.split("(?<=\\*)|(?=\\*)"); 366 StringBuilder builder = new StringBuilder(); 367 for (String token : tokens) { 368 if ("*".equals(token)) { 369 builder.append(".*"); 370 } else if (token.length() > 0) { 371 builder.append(Pattern.quote(token)); 372 } 373 } 374 375 return builder.toString(); 376 } 377 378 public void run() { 379 testng.run(); 380 } 381 382 public int getStatus() { 383 return testng.getStatus(); 384 } 385 386 public String getOutputDir() { 387 return outputDir; 388 } 389 390 public static CommandLine getCommandLine(Options options, String[] args){ 391 CommandLineParser parser = new BasicParser(); 392 CommandLine cmd = null; 393 try { 394 cmd = parser.parse(options, args); 395 } catch (ParseException e) { 396 System.err.println("ERROR: " + e.getLocalizedMessage()); 397 printUsage(options); 398 } 399 400 if (cmd.hasOption("help")) { 401 printUsage(options); 402 } 403 return cmd; 404 } 405 406 public static void executeTestSuite(String[] args, Options options, String reportTitle) { 407 executeTestSuite(args, options, reportTitle, Collections.<XmlClass>emptyList()); 408 } 409 public static void executeTestSuite(String[] args, Options options, String reportTitle, List<XmlClass> classes) { 410 // actual test suite execution 411 try { 412 CommandLine cmd = LdpTestSuite.getCommandLine(options, args); 413 LdpTestSuite ldpTestSuite = new LdpTestSuite(cmd, reportTitle); 414 ldpTestSuite.addTestClasses(classes); 415 ldpTestSuite.run(); 416 System.exit(ldpTestSuite.getStatus()); 417 } catch (Exception e) { 418 e.printStackTrace(); 419 Throwable cause = ExceptionUtils.getRootCause(e); 420 System.err.println("ERROR: " + (cause != null ? cause.getMessage() : e.getMessage())); 421 printUsage(options); 422 } 423 } 424 425 426 private static void printUsage(Options options) { 427 HelpFormatter formatter = new HelpFormatter(); 428 formatter.setOptionComparator(new Comparator<Option>() { 429 @Override 430 public int compare(Option o1, Option o2) { 431 if ("server".equals(o1.getLongOpt())) { 432 return -10000; 433 } else if ("help".equals(o1.getLongOpt())) { 434 return 10000; 435 } else { 436 return o1.getLongOpt().compareTo(o2.getLongOpt()); 437 } 438 } 439 }); 440 System.out.println(); 441 formatter.printHelp("java -jar ldp-testsuite.jar", options); 442 System.out.println(); 443 System.exit(-1); 444 } 445 446 private static void printEarlUsage(String missingArg) { 447 System.out.println("--earl missing arg: "+missingArg); 448 System.out.println("Required additional args:"); 449 for (String arg: EARLDEPEDENTARGS) { 450 System.out.println("\t--"+arg); 451 } 452 System.exit(1); 453 } 454 455 @SuppressWarnings("static-access") 456 public static OptionGroup addCommonOptions() { 457 OptionGroup common = new OptionGroup(); 458 common.addOption(OptionBuilder.withLongOpt("server") 459 .withDescription("server url to run the test suite").hasArg() 460 .withArgName("server").isRequired().create()); 461 462 common.addOption(OptionBuilder.withLongOpt("auth") 463 .withDescription("server basic authentication credentials following the syntax username:password").hasArg() 464 .withArgName("username:password").create()); 465 466 common.addOption(OptionBuilder.withLongOpt("earl") 467 .withDescription("General EARL ttl file").withArgName("earl") 468 .isRequired(false).create()); 469 470 common.addOption(OptionBuilder.withLongOpt("includedGroups") 471 .withDescription("test groups to run, separated by a space").hasArgs() 472 .withArgName("includedGroups").isRequired(false) 473 .create()); 474 475 common.addOption(OptionBuilder.withLongOpt("excludedGroups") 476 .withDescription("test groups to not run, separated by a space").hasArgs() 477 .withArgName("excludedGroups").isRequired(false) 478 .create()); 479 480 common.addOption(OptionBuilder.withLongOpt("test") 481 .withDescription("which tests to run (* is a wildcard)") 482 .hasArgs().withArgName("test names") 483 .create()); 484 485 common.addOption(OptionBuilder.withLongOpt("output") 486 .withDescription("output directory (current directory by default)") 487 .hasArgs().withArgName("output") 488 .isRequired(false).create()); 489 490 common.addOption(OptionBuilder.withLongOpt("httpLogging") 491 .withDescription("log HTTP requests and responses on validation failures") 492 .isRequired(false).create()); 493 494 common.addOption(OptionBuilder.withLongOpt("skipLogging") 495 .withDescription("log skip test messages") 496 .isRequired(false).create()); 497 498 common.addOption(OptionBuilder.withLongOpt("help") 499 .withDescription("prints this usage help") 500 .isRequired(false).create()); 501 return common; 502 } 503 504 @SuppressWarnings("static-access") 505 public static OptionGroup addEarlOptions() { 506 OptionGroup earl = new OptionGroup(); 507 // --earl dependent values 508 earl.addOption(OptionBuilder 509 .withLongOpt("software") 510 .withDescription( 511 "title of the software test suite runs on: required with --earl") 512 .hasArg().withArgName("software").isRequired(false).create()); 513 514 earl.addOption(OptionBuilder 515 .withLongOpt("developer") 516 .withDescription( 517 "the name of the software developer: required with --earl") 518 .hasArg().withArgName("dev-name").isRequired(false).create()); 519 520 earl.addOption(OptionBuilder 521 .withLongOpt("mbox") 522 .withDescription( 523 "email of the sofware developer: optional with --earl") 524 .hasArg().withArgName("mbox").isRequired(false).create()); 525 526 earl.addOption(OptionBuilder 527 .withLongOpt("language") 528 .withDescription( 529 "primary programming language of the software: required with --earl") 530 .hasArg().withArgName("language").isRequired(false).create()); 531 532 earl.addOption(OptionBuilder 533 .withLongOpt("homepage") 534 .withDescription( 535 "the homepage of the suite runs against: required with --earl") 536 .hasArg().withArgName("homepage").isRequired(false).create()); 537 538 earl.addOption(OptionBuilder 539 .withLongOpt("assertor") 540 .withDescription( 541 "the URL of the person or agent that asserts the results: required with --earl") 542 .hasArg().withArgName("assertor").isRequired(false).create()); 543 544 earl.addOption(OptionBuilder.withLongOpt("shortname") 545 .withDescription("a simple short name: required with --earl") 546 .hasArg().withArgName("shortname").isRequired(false).create()); 547 // end of --earl dependent values 548 return earl; 549 } 550 551}