Skip to content

Commit

Permalink
Fix bug where crawler plugin invocation couldn't find crawler file pr…
Browse files Browse the repository at this point in the history 
…operly. And other minor cleanup.
  • Loading branch information
@davewichers
davewichers committed Sep 23, 2021
1 parent 014a1a9 commit 99f5395
Show file tree
Hide file tree
Showing 5 changed files with 79 additions and 69 deletions.
1 change: 1 addition & 0 deletions examplescripts_configfiles/runCrawler.sh
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# An EXAMPLE of how to run the crawler on a TARGET test suite using the Benchmark Maven Plugin
source "scripts/verifyBenchmarkPluginAvailable.sh"
mvn org.owasp:benchmarkutils-maven-plugin:run-crawler -DcrawlerFile=data/TESTSUITENAME-crawler-http.xml

10 changes: 2 additions & 8 deletions plugin/src/main/java/org/owasp/benchmarkutils/helpers/Utils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,6 @@ public class Utils {

public static final String DATA_DIR = USERDIR + "data" + File.separator;

public static final String CRAWLER_CONFIG_FILE = "benchmark-attack-http.xml";

public static final DocumentBuilderFactory safeDocBuilderFactory =
DocumentBuilderFactory.newInstance();

Expand Down Expand Up @@ -97,10 +95,7 @@ public static File getFileFromClasspath(String filePath, ClassLoader classLoader
System.out.printf(
"getFileFromClasspath() url.toURI() is: %s and external form is: %s%n",
resourceURI, externalFormURI);
// String filePath = resourceURI.getPath();
// System.out.println("getFileFromClasspath() url.toURI().getPath()
// is: " + filePath);
// if (resourceURI != null) return new File(resourceURI);

if (externalFormURI != null) return new File(externalFormURI);
else {
System.out.printf(
Expand Down Expand Up @@ -184,10 +179,9 @@ public static TestSuite parseHttpFile(File file) throws JAXBException, FileNotFo

TestSuite testSuite = null;
JAXBContext context = JAXBContextFactory.createContext(new Class[] {TestSuite.class}, null);
String crawlerFileName = new File(Utils.DATA_DIR, CRAWLER_CONFIG_FILE).getPath();
Unmarshaller unmarshaller = context.createUnmarshaller();
unmarshaller.setEventHandler(new javax.xml.bind.helpers.DefaultValidationEventHandler());
testSuite = (TestSuite) unmarshaller.unmarshal(new FileReader(crawlerFileName));
testSuite = (TestSuite) unmarshaller.unmarshal(new FileReader(file));

return testSuite;
}
Expand Down
1 change: 1 addition & 0 deletions plugin/src/main/java/org/owasp/benchmarkutils/score/BenchmarkScore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,7 @@ private static boolean processCommandLineArgs(String[] args) {
return allParamsOK;
}

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
// The Maven plugin invocation of this can have configFile be null, so we check for that
// specifically
Expand Down
96 changes: 58 additions & 38 deletions plugin/src/main/java/org/owasp/benchmarkutils/tools/BenchmarkCrawler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,23 +63,38 @@
public class BenchmarkCrawler extends AbstractMojo {

@Parameter(property = "crawlerFile")
// TODO: Utils.DATA_DIR is not actually a constant!
String crawlerFileName = new File(Utils.DATA_DIR, "benchmark-crawler-http.xml").getPath();
String pluginFilenameParam;

File crawlerFile = new File(crawlerFileName); // default location;
/*
* Attaching the @Parameter property to the crawlerFile variable directly didn't work for some
* reason. So I attached it to a new String variable, and set it later. No clue why it doesn't
* work. But for now, leaving it this way because it works.
*
* If you run the mvn command with -X, when invoking this plugin, you'd see something like
* this at the end:
*
* [DEBUG] (s) crawlerFile = /Users/PATH/TO/BenchmarkJava/data/benchmark-crawler-http.xml
* [DEBUG] -- end configuration --
* but the crawlerFile variable would be null.
*
* When it should be:
* [DEBUG] (f) crawlerFile = data/benchmark-crawler-http.xml
* [DEBUG] -- end configuration --
*
* So after changing this, I now get:
* [DEBUG] (f) pluginFilenameParam = data/benchmark-crawler-http.xml
* [DEBUG] -- end configuration --
* and the pluginFilenameParam variable value is set properly.
*/
String crawlerFile;

File theCrawlerFile;
String selectedTestCaseName = null;

TestSuite testSuite;

BenchmarkCrawler() {
// Default constructor required for to support Maven plugin API.
// The BenchmarkCrawler(File) must eventually be used before run() is invoked on that
// instance of the Crawler.
}

BenchmarkCrawler(File file) {
this.crawlerFile = file;
// A default constructor required to support Maven plugin API.
// The theCrawlerFile has to be instantiated before a crawl can be done.
}

/** Crawl the target test suite. */
Expand All @@ -94,36 +109,47 @@ protected void run() {

void load() {
try {
// Force initialization of the Categories singleton.
InputStream categoriesFileStream =
BenchmarkScore.class.getClassLoader().getResourceAsStream(Categories.FILENAME);
new Categories(categoriesFileStream);

testSuite = Utils.parseHttpFile(crawlerFile);
this.testSuite = Utils.parseHttpFile(this.theCrawlerFile);
Collections.sort(
testSuite.getTestCases(),
this.testSuite.getTestCases(),
AbstractTestCaseRequest.getNameComparator()); // Probably not necessary

// This allows a single test case to be tested, rather than all of them.
if (selectedTestCaseName != null) {
for (AbstractTestCaseRequest request : testSuite.getTestCases()) {
for (AbstractTestCaseRequest request : this.testSuite.getTestCases()) {
if (request.getName().equals(selectedTestCaseName)) {
List<AbstractTestCaseRequest> requests = new ArrayList<>();
requests.add(request);
testSuite = new TestSuite();
testSuite.setTestCases(requests);
this.testSuite = new TestSuite();
this.testSuite.setTestCases(requests);
break;
}
}
}
} catch (Exception e) {
System.out.println("ERROR: Problem with specified crawler file: " + crawlerFile);
System.out.println(
"ERROR: Problem with specified crawler file: " + this.theCrawlerFile);
e.printStackTrace();
System.exit(-1);
}
}

public void setCrawlerFile(File crawlerFile) {
this.crawlerFile = crawlerFile;
public void setCrawlerFile(File theCrawlerFile) {
this.theCrawlerFile = theCrawlerFile;
}

/**
* This method could be static, but needs to be an instance method so Verification crawler can
* overrite this method.
*
* @param testSuite The TestSuite to crawl.
* @throws Exception
*/
protected void crawl(TestSuite testSuite) throws Exception {
CloseableHttpClient httpclient = createAcceptSelfSignedCertificateClient();
long start = System.currentTimeMillis();
Expand All @@ -133,12 +159,7 @@ protected void crawl(TestSuite testSuite) throws Exception {
HttpUriRequest request = requestTemplate.buildAttackRequest();

// Send the next test case request
try {
sendRequest(httpclient, request);
} catch (Exception e) {
System.err.println("\n FAILED: " + e.getMessage());
e.printStackTrace();
}
sendRequest(httpclient, request);
}

// Log the elapsed time for all test cases
Expand Down Expand Up @@ -182,7 +203,6 @@ static CloseableHttpClient createAcceptSelfSignedCertificateClient()
*
* @param httpclient - The HTTP client to use to make the request
* @param request - THe HTTP request to issue
* @throws IOException
*/
static ResponseInfo sendRequest(CloseableHttpClient httpclient, HttpUriRequest request) {
ResponseInfo responseInfo = new ResponseInfo();
Expand Down Expand Up @@ -259,8 +279,14 @@ private void processCommandLineArgs(String[] args) {
CommandLine line = parser.parse(options, args);

if (line.hasOption("f")) {
crawlerFileName = line.getOptionValue("f");
crawlerFile = new File(crawlerFileName);
this.crawlerFile = line.getOptionValue("f");
File targetFile = new File(this.crawlerFile);
if (targetFile.exists()) {
setCrawlerFile(targetFile);
} else {
throw new RuntimeException(
"Could not find crawler configuration file '" + this.crawlerFile + "'");
}
}
if (line.hasOption("h")) {
formatter.printHelp("BenchmarkCrawlerVerification", options, true);
Expand All @@ -272,20 +298,14 @@ private void processCommandLineArgs(String[] args) {
formatter.printHelp("BenchmarkCrawler", options);
throw new RuntimeException("Error parsing arguments: ", e);
}

if (crawlerFile.exists()) {
setCrawlerFile(new File(crawlerFileName));
} else {
throw new RuntimeException(
"Could not find crawler configuration file '" + crawlerFileName + "'");
}
}

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
if (null == crawlerFileName) {
System.out.println("ERROR: A crawler file must be specified.");
if (null == this.pluginFilenameParam) {
System.out.println("ERROR: A crawlerFile parameter must be specified.");
} else {
String[] mainArgs = {"-f", crawlerFileName};
String[] mainArgs = {"-f", this.pluginFilenameParam};
main(mainArgs);
}
}
Expand Down
40 changes: 17 additions & 23 deletions plugin/src/main/java/org/owasp/benchmarkutils/tools/BenchmarkCrawlerVerification.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,8 @@ public class BenchmarkCrawlerVerification extends BenchmarkCrawler {
SimpleFileLogger uLogger;

BenchmarkCrawlerVerification() {
// Default constructor required to support Maven plugin API.
// The BenchmarkCrawlerVerification(File) must eventually be used before run()
// is invoked on that instance of the Crawler.
}

BenchmarkCrawlerVerification(File file) {
super(file);
// A default constructor required to support Maven plugin API.
// The theCrawlerFile has to be instantiated before a crawl can be done.
}

@Override
Expand Down Expand Up @@ -272,9 +267,9 @@ protected static void handleResponse(TestCaseVerificationResults result)
*/
private void processCommandLineArgs(String[] args) {

// Set default location
// Set default attack crawler file
String crawlerFileName = new File(Utils.DATA_DIR, "benchmark-attack-http.xml").getPath();
File crawlerFile = new File(crawlerFileName);
this.theCrawlerFile = new File(crawlerFileName);

RegressionTesting.isTestingEnabled = true;

Expand Down Expand Up @@ -312,8 +307,16 @@ private void processCommandLineArgs(String[] args) {
CommandLine line = parser.parse(options, args);

if (line.hasOption("f")) {
crawlerFileName = line.getOptionValue("f");
crawlerFile = new File(crawlerFileName);
this.crawlerFile = line.getOptionValue("f");
File targetFile = new File(this.crawlerFile);
if (targetFile.exists()) {
setCrawlerFile(targetFile);
// Crawler output files go into the same directory as the crawler config file
CRAWLER_DATA_DIR = targetFile.getParent() + File.separator;
} else {
throw new RuntimeException(
"Could not find crawler configuration file '" + this.crawlerFile + "'");
}
}
if (line.hasOption("h")) {
formatter.printHelp("BenchmarkCrawlerVerification", options, true);
Expand All @@ -328,23 +331,14 @@ private void processCommandLineArgs(String[] args) {
formatter.printHelp("BenchmarkCrawlerVerification", options);
throw new RuntimeException("Error parsing arguments: ", e);
}

if (crawlerFile.exists()) {
setCrawlerFile(new File(crawlerFileName));
// Crawler output files go into the same directory as the crawler config file
CRAWLER_DATA_DIR = crawlerFile.getParent() + File.separator;
} else {
throw new RuntimeException(
"Could not find crawler configuration file '" + crawlerFileName + "'");
}
}

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
if (null == crawlerFileName) {
System.out.println("ERROR: A verification crawler file must be specified.");
if (null == this.pluginFilenameParam) {
System.out.println("ERROR: A crawlerFile parameter must be specified.");
} else {
String[] mainArgs = {"-f", crawlerFileName};
String[] mainArgs = {"-f", this.pluginFilenameParam};
main(mainArgs);
}
}
Expand Down

0 comments on commit 99f5395

Please sign in to comment.