updated and added test cases

This commit is contained in:
Jeremy Long
2017-03-11 11:09:31 -05:00
parent a61bba2f72
commit 46f227e92e
2 changed files with 90 additions and 7 deletions

View File

@@ -47,8 +47,8 @@ public final class UrlStringUtils {
private static final Pattern IS_URL_TEST = Pattern.compile("^(ht|f)tps?://.*", Pattern.CASE_INSENSITIVE);
/**
* Tests if the text provided contains a URL. This is somewhat limited search in that it only looks for
* (ftp|http|https)://
* Tests if the text provided contains a URL. This is somewhat limited
* search in that it only looks for (ftp|http|https)://
*
* @param text the text to search
* @return true if the text contains a url, otherwise false
@@ -67,14 +67,16 @@ public final class UrlStringUtils {
return IS_URL_TEST.matcher(text).matches();
}
/**
* A listing of domain parts that should not be used as evidence. Yes, this is an incomplete list.
* A listing of domain parts that should not be used as evidence. Yes, this
* is an incomplete list.
*/
private static final Set<String> IGNORE_LIST = new HashSet<>(
Arrays.asList("www", "com", "org", "gov", "info", "name", "net", "pro", "tel", "mobi", "xxx"));
/**
* <p>
* Takes a URL, in String format, and adds the important parts of the URL to a list of strings.</p>
* Takes a URL, in String format, and adds the important parts of the URL to
* a list of strings.</p>
* <p>
* Example, given the following input:</p>
* <code>"https://www.somedomain.com/path1/path2/file.php?id=439"</code>
@@ -99,14 +101,21 @@ public final class UrlStringUtils {
}
final String document = url.getPath();
final String[] pathParts = document.split("[\\//]");
for (int i = 0; i < pathParts.length - 2; i++) {
for (int i = 0; i < pathParts.length - 1; i++) {
if (!pathParts[i].isEmpty()) {
importantParts.add(pathParts[i]);
}
}
if (pathParts.length > 0 && !pathParts[pathParts.length - 1].isEmpty()) {
final String fileNameNoExt = pathParts[pathParts.length - 1].replaceAll("\\..*{0,5}$", "");
importantParts.add(fileNameNoExt);
final String tmp = pathParts[pathParts.length - 1];
final int pos = tmp.lastIndexOf('.');
if (pos > 1) {
importantParts.add(tmp.substring(0, pos));
} else if (pos == 0 && tmp.length()>1) {
importantParts.add(tmp.substring(1));
} else {
importantParts.add(tmp);
}
}
return importantParts;
}