Initial commit

This commit is contained in:
Šesták Vít
2016-01-10 17:31:07 +01:00
commit 4b87ced31f
104 changed files with 4870 additions and 0 deletions

8
.gitignore vendored Normal file
View File

@@ -0,0 +1,8 @@
/RUNNING_PID
/logs/
/project/*-shim.sbt
/project/project/
/project/target/
/target/
.idea
conf/application.conf

15
README.md Normal file
View File

@@ -0,0 +1,15 @@
## Database
I decided to use PostgreSQL, because
* It is easy to set up
* It is reasonably strict by default. For example, when you try to insert a 256-character string in varchar(64), you get an error. (In MySQL, it gets silently truncated by default!)
## TODO
### Naming
* Library × Identifier × PlainLibraryIdentifier should be renamed
* Identifier is the most verbose one, it comes from OWASP Dependency Check.
* Library is a record stored in our database.
* PlainLibraryIdentifier is just version-less (e.g. `"$groupId:$artifactId"`) library identifier.

334
activator vendored Executable file
View File

@@ -0,0 +1,334 @@
#!/usr/bin/env bash
### ------------------------------- ###
### Helper methods for BASH scripts ###
### ------------------------------- ###
realpath () {
(
TARGET_FILE="$1"
cd "$(dirname "$TARGET_FILE")"
TARGET_FILE=$(basename "$TARGET_FILE")
COUNT=0
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
do
TARGET_FILE=$(readlink "$TARGET_FILE")
cd "$(dirname "$TARGET_FILE")"
TARGET_FILE=$(basename "$TARGET_FILE")
COUNT=$(($COUNT + 1))
done
if [ "$TARGET_FILE" == "." -o "$TARGET_FILE" == ".." ]; then
cd "$TARGET_FILE"
TARGET_FILEPATH=
else
TARGET_FILEPATH=/$TARGET_FILE
fi
# make sure we grab the actual windows path, instead of cygwin's path.
if ! is_cygwin; then
echo "$(pwd -P)/$TARGET_FILE"
else
echo $(cygwinpath "$(pwd -P)/$TARGET_FILE")
fi
)
}
# TODO - Do we need to detect msys?
# Uses uname to detect if we're in the odd cygwin environment.
is_cygwin() {
local os=$(uname -s)
case "$os" in
CYGWIN*) return 0 ;;
*) return 1 ;;
esac
}
# This can fix cygwin style /cygdrive paths so we get the
# windows style paths.
cygwinpath() {
local file="$1"
if is_cygwin; then
echo $(cygpath -w $file)
else
echo $file
fi
}
# Make something URI friendly
make_url() {
url="$1"
local nospaces=${url// /%20}
if is_cygwin; then
echo "/${nospaces//\\//}"
else
echo "$nospaces"
fi
}
# Detect if we should use JAVA_HOME or just try PATH.
get_java_cmd() {
if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
echo "$JAVA_HOME/bin/java"
else
echo "java"
fi
}
echoerr () {
echo 1>&2 "$@"
}
vlog () {
[[ $verbose || $debug ]] && echoerr "$@"
}
dlog () {
[[ $debug ]] && echoerr "$@"
}
execRunner () {
# print the arguments one to a line, quoting any containing spaces
[[ $verbose || $debug ]] && echo "# Executing command line:" && {
for arg; do
if printf "%s\n" "$arg" | grep -q ' '; then
printf "\"%s\"\n" "$arg"
else
printf "%s\n" "$arg"
fi
done
echo ""
}
exec "$@"
}
addJava () {
dlog "[addJava] arg = '$1'"
java_args=( "${java_args[@]}" "$1" )
}
addApp () {
dlog "[addApp] arg = '$1'"
sbt_commands=( "${app_commands[@]}" "$1" )
}
addResidual () {
dlog "[residual] arg = '$1'"
residual_args=( "${residual_args[@]}" "$1" )
}
addDebugger () {
addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
}
addConfigOpts () {
dlog "[addConfigOpts] arg = '$*'"
for item in $*
do
addJava "$item"
done
}
# a ham-fisted attempt to move some memory settings in concert
# so they need not be messed around with individually.
get_mem_opts () {
local mem=${1:-1024}
local meta=$(( $mem / 4 ))
(( $meta > 256 )) || meta=256
(( $meta < 1024 )) || meta=1024
# default is to set memory options but this can be overridden by code section below
memopts="-Xms${mem}m -Xmx${mem}m"
if [[ "${java_version}" > "1.8" ]]; then
extmemopts="-XX:MetaspaceSize=64m -XX:MaxMetaspaceSize=${meta}m"
else
extmemopts="-XX:PermSize=64m -XX:MaxPermSize=${meta}m"
fi
if [[ "${java_opts}" == *-Xmx* ]] || [[ "${java_opts}" == *-Xms* ]] || [[ "${java_opts}" == *-XX:MaxPermSize* ]] || [[ "${java_opts}" == *-XX:ReservedCodeCacheSize* ]] || [[ "${java_opts}" == *-XX:MaxMetaspaceSize* ]]; then
# if we detect any of these settings in ${java_opts} we need to NOT output our settings.
# The reason is the Xms/Xmx, if they don't line up, cause errors.
memopts=""
extmemopts=""
fi
echo "${memopts} ${extmemopts}"
}
require_arg () {
local type="$1"
local opt="$2"
local arg="$3"
if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
die "$opt requires <$type> argument"
fi
}
is_function_defined() {
declare -f "$1" > /dev/null
}
# If we're *not* running in a terminal, and we don't have any arguments, then we need to add the 'ui' parameter
detect_terminal_for_ui() {
[[ ! -t 0 ]] && [[ "${#residual_args}" == "0" ]] && {
addResidual "ui"
}
# SPECIAL TEST FOR MAC
[[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]] && [[ "${#residual_args}" == "0" ]] && {
echo "Detected MAC OSX launched script...."
echo "Swapping to UI"
addResidual "ui"
}
}
# Processes incoming arguments and places them in appropriate global variables. called by the run method.
process_args () {
while [[ $# -gt 0 ]]; do
case "$1" in
-h|-help) usage; exit 1 ;;
-v|-verbose) verbose=1 && shift ;;
-d|-debug) debug=1 && shift ;;
-mem) require_arg integer "$1" "$2" && app_mem="$2" && shift 2 ;;
-jvm-debug)
if echo "$2" | grep -E ^[0-9]+$ > /dev/null; then
addDebugger "$2" && shift
else
addDebugger 9999
fi
shift ;;
-java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;;
-D*) addJava "$1" && shift ;;
-J*) addJava "${1:2}" && shift ;;
*) addResidual "$1" && shift ;;
esac
done
is_function_defined process_my_args && {
myargs=("${residual_args[@]}")
residual_args=()
process_my_args "${myargs[@]}"
}
}
# Actually runs the script.
run() {
# TODO - check for sane environment
# process the combined args, then reset "$@" to the residuals
process_args "$@"
detect_terminal_for_ui
set -- "${residual_args[@]}"
argumentCount=$#
#check for jline terminal fixes on cygwin
if is_cygwin; then
stty -icanon min 1 -echo > /dev/null 2>&1
addJava "-Djline.terminal=jline.UnixTerminal"
addJava "-Dsbt.cygwin=true"
fi
# run sbt
execRunner "$java_cmd" \
"-Dactivator.home=$(make_url "$activator_home")" \
$(get_mem_opts $app_mem) \
${java_opts[@]} \
${java_args[@]} \
-jar "$app_launcher" \
"${app_commands[@]}" \
"${residual_args[@]}"
local exit_code=$?
if is_cygwin; then
stty icanon echo > /dev/null 2>&1
fi
exit $exit_code
}
# Loads a configuration file full of default command line options for this script.
loadConfigFile() {
cat "$1" | sed '/^\#/d'
}
### ------------------------------- ###
### Start of customized settings ###
### ------------------------------- ###
usage() {
cat <<EOM
Usage: $script_name <command> [options]
Command:
ui Start the Activator UI
new [name] [template-id] Create a new project with [name] using template [template-id]
list-templates Print all available template names
-h | -help Print this message
Options:
-v | -verbose Make this runner chattier
-d | -debug Set sbt log level to debug
-mem <integer> Set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem))
-jvm-debug <port> Turn on JVM debugging, open at the given port.
# java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
-java-home <path> Alternate JAVA_HOME
# jvm options and output control
-Dkey=val Pass -Dkey=val directly to the java runtime
-J-X Pass option -X directly to the java runtime
(-J is stripped)
# environment variables (read from context)
JAVA_OPTS Environment variable, if unset uses ""
SBT_OPTS Environment variable, if unset uses ""
ACTIVATOR_OPTS Environment variable, if unset uses ""
In the case of duplicated or conflicting options, the order above
shows precedence: environment variables lowest, command line options highest.
EOM
}
### ------------------------------- ###
### Main script ###
### ------------------------------- ###
declare -a residual_args
declare -a java_args
declare -a app_commands
declare -r real_script_path="$(realpath "$0")"
declare -r activator_home="$(realpath "$(dirname "$real_script_path")")"
declare -r app_version="1.3.3"
declare -r app_launcher="${activator_home}/activator-launch-${app_version}.jar"
declare -r script_name=activator
java_cmd=$(get_java_cmd)
declare -r java_opts=( "${ACTIVATOR_OPTS[@]}" "${SBT_OPTS[@]}" "${JAVA_OPTS[@]}" "${java_opts[@]}" )
userhome="$HOME"
if is_cygwin; then
# cygwin sets home to something f-d up, set to real windows homedir
userhome="$USERPROFILE"
fi
declare -r activator_user_home_dir="${userhome}/.activator"
declare -r java_opts_config_home="${activator_user_home_dir}/activatorconfig.txt"
declare -r java_opts_config_version="${activator_user_home_dir}/${app_version}/activatorconfig.txt"
# Now check to see if it's a good enough version
declare -r java_version=$("$java_cmd" -version 2>&1 | awk -F '"' '/version/ {print $2}')
if [[ "$java_version" == "" ]]; then
echo
echo No java installations was detected.
echo Please go to http://www.java.com/getjava/ and download
echo
exit 1
elif [[ ! "$java_version" > "1.6" ]]; then
echo
echo The java installation you have is not up to date
echo Activator requires at least version 1.6+, you have
echo version $java_version
echo
echo Please go to http://www.java.com/getjava/ and download
echo a valid Java Runtime and install before running Activator.
echo
exit 1
fi
# if configuration files exist, prepend their contents to the java args so it can be processed by this runner
# a "versioned" config trumps one on the top level
if [[ -f "$java_opts_config_version" ]]; then
addConfigOpts $(loadConfigFile "$java_opts_config_version")
elif [[ -f "$java_opts_config_home" ]]; then
addConfigOpts $(loadConfigFile "$java_opts_config_home")
fi
run "$@"

BIN
activator-launch-1.3.3.jar Normal file

Binary file not shown.

231
activator.bat vendored Normal file
View File

@@ -0,0 +1,231 @@
@REM activator launcher script
@REM
@REM Environment:
@REM In order for Activator to work you must have Java available on the classpath
@REM JAVA_HOME - location of a JDK home dir (optional if java on path)
@REM CFG_OPTS - JVM options (optional)
@REM Configuration:
@REM activatorconfig.txt found in the ACTIVATOR_HOME or ACTIVATOR_HOME/ACTIVATOR_VERSION
@setlocal enabledelayedexpansion
@echo off
set "var1=%~1"
if defined var1 (
if "%var1%"=="help" (
echo.
echo Usage activator [options] [command]
echo.
echo Commands:
echo ui Start the Activator UI
echo new [name] [template-id] Create a new project with [name] using template [template-id]
echo list-templates Print all available template names
echo help Print this message
echo.
echo Options:
echo -jvm-debug [port] Turn on JVM debugging, open at the given port. Defaults to 9999 if no port given.
echo.
echo Environment variables ^(read from context^):
echo JAVA_OPTS Environment variable, if unset uses ""
echo SBT_OPTS Environment variable, if unset uses ""
echo ACTIVATOR_OPTS Environment variable, if unset uses ""
echo.
echo Please note that in order for Activator to work you must have Java available on the classpath
echo.
goto :end
)
)
if "%ACTIVATOR_HOME%"=="" (
set "ACTIVATOR_HOME=%~dp0"
@REM remove trailing "\" from path
set ACTIVATOR_HOME=!ACTIVATOR_HOME:~0,-1!
)
set ERROR_CODE=0
set APP_VERSION=1.3.3
set ACTIVATOR_LAUNCH_JAR=activator-launch-%APP_VERSION%.jar
rem Detect if we were double clicked, although theoretically A user could
rem manually run cmd /c
for %%x in (%cmdcmdline%) do if %%~x==/c set DOUBLECLICKED=1
rem FIRST we load a config file of extra options (if there is one)
set "CFG_FILE_HOME=%UserProfile%\.activator\activatorconfig.txt"
set "CFG_FILE_VERSION=%UserProfile%\.activator\%APP_VERSION%\activatorconfig.txt"
set CFG_OPTS=
if exist %CFG_FILE_VERSION% (
FOR /F "tokens=* eol=# usebackq delims=" %%i IN ("%CFG_FILE_VERSION%") DO (
set DO_NOT_REUSE_ME=%%i
rem ZOMG (Part #2) WE use !! here to delay the expansion of
rem CFG_OPTS, otherwise it remains "" for this loop.
set CFG_OPTS=!CFG_OPTS! !DO_NOT_REUSE_ME!
)
)
if "%CFG_OPTS%"=="" (
if exist %CFG_FILE_HOME% (
FOR /F "tokens=* eol=# usebackq delims=" %%i IN ("%CFG_FILE_HOME%") DO (
set DO_NOT_REUSE_ME=%%i
rem ZOMG (Part #2) WE use !! here to delay the expansion of
rem CFG_OPTS, otherwise it remains "" for this loop.
set CFG_OPTS=!CFG_OPTS! !DO_NOT_REUSE_ME!
)
)
)
rem We use the value of the JAVACMD environment variable if defined
set _JAVACMD=%JAVACMD%
if "%_JAVACMD%"=="" (
if not "%JAVA_HOME%"=="" (
if exist "%JAVA_HOME%\bin\java.exe" set "_JAVACMD=%JAVA_HOME%\bin\java.exe"
rem if there is a java home set we make sure it is the first picked up when invoking 'java'
SET "PATH=%JAVA_HOME%\bin;%PATH%"
)
)
if "%_JAVACMD%"=="" set _JAVACMD=java
rem Detect if this java is ok to use.
for /F %%j in ('"%_JAVACMD%" -version 2^>^&1') do (
if %%~j==java set JAVAINSTALLED=1
if %%~j==openjdk set JAVAINSTALLED=1
)
rem Detect the same thing about javac
if "%_JAVACCMD%"=="" (
if not "%JAVA_HOME%"=="" (
if exist "%JAVA_HOME%\bin\javac.exe" set "_JAVACCMD=%JAVA_HOME%\bin\javac.exe"
)
)
if "%_JAVACCMD%"=="" set _JAVACCMD=javac
for /F %%j in ('"%_JAVACCMD%" -version 2^>^&1') do (
if %%~j==javac set JAVACINSTALLED=1
)
rem BAT has no logical or, so we do it OLD SCHOOL! Oppan Redmond Style
set JAVAOK=true
if not defined JAVAINSTALLED set JAVAOK=false
if not defined JAVACINSTALLED set JAVAOK=false
if "%JAVAOK%"=="false" (
echo.
echo A Java JDK is not installed or can't be found.
if not "%JAVA_HOME%"=="" (
echo JAVA_HOME = "%JAVA_HOME%"
)
echo.
echo Please go to
echo http://www.oracle.com/technetwork/java/javase/downloads/index.html
echo and download a valid Java JDK and install before running Activator.
echo.
echo If you think this message is in error, please check
echo your environment variables to see if "java.exe" and "javac.exe" are
echo available via JAVA_HOME or PATH.
echo.
if defined DOUBLECLICKED pause
exit /B 1
)
rem Check what Java version is being used to determine what memory options to use
for /f "tokens=3" %%g in ('java -version 2^>^&1 ^| findstr /i "version"') do (
set JAVA_VERSION=%%g
)
rem Strips away the " characters
set JAVA_VERSION=%JAVA_VERSION:"=%
rem TODO Check if there are existing mem settings in JAVA_OPTS/CFG_OPTS and use those instead of the below
for /f "delims=. tokens=1-3" %%v in ("%JAVA_VERSION%") do (
set MAJOR=%%v
set MINOR=%%w
set BUILD=%%x
set META_SIZE=-XX:MetaspaceSize=64M -XX:MaxMetaspaceSize=256M
if "!MINOR!" LSS "8" (
set META_SIZE=-XX:PermSize=64M -XX:MaxPermSize=256M
)
set MEM_OPTS=!META_SIZE!
)
rem We use the value of the JAVA_OPTS environment variable if defined, rather than the config.
set _JAVA_OPTS=%JAVA_OPTS%
if "%_JAVA_OPTS%"=="" set _JAVA_OPTS=%CFG_OPTS%
set DEBUG_OPTS=
rem Loop through the arguments, building remaining args in args variable
set args=
:argsloop
if not "%~1"=="" (
rem Checks if the argument contains "-D" and if true, adds argument 1 with 2 and puts an equal sign between them.
rem This is done since batch considers "=" to be a delimiter so we need to circumvent this behavior with a small hack.
set arg1=%~1
if "!arg1:~0,2!"=="-D" (
set "args=%args% "%~1"="%~2""
shift
shift
goto argsloop
)
if "%~1"=="-jvm-debug" (
if not "%~2"=="" (
rem This piece of magic somehow checks that an argument is a number
for /F "delims=0123456789" %%i in ("%~2") do (
set var="%%i"
)
if defined var (
rem Not a number, assume no argument given and default to 9999
set JPDA_PORT=9999
) else (
rem Port was given, shift arguments
set JPDA_PORT=%~2
shift
)
) else (
set JPDA_PORT=9999
)
shift
set DEBUG_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=!JPDA_PORT!
goto argsloop
)
rem else
set "args=%args% "%~1""
shift
goto argsloop
)
:run
if "!args!"=="" (
if defined DOUBLECLICKED (
set CMDS="ui"
) else set CMDS=!args!
) else set CMDS=!args!
rem We add a / in front, so we get file:///C: instead of file://C:
rem Java considers the later a UNC path.
rem We also attempt a solid effort at making it URI friendly.
rem We don't even bother with UNC paths.
set JAVA_FRIENDLY_HOME_1=/!ACTIVATOR_HOME:\=/!
set JAVA_FRIENDLY_HOME=/!JAVA_FRIENDLY_HOME_1: =%%20!
rem Checks if the command contains spaces to know if it should be wrapped in quotes or not
set NON_SPACED_CMD=%_JAVACMD: =%
if "%_JAVACMD%"=="%NON_SPACED_CMD%" %_JAVACMD% %DEBUG_OPTS% %MEM_OPTS% %ACTIVATOR_OPTS% %SBT_OPTS% %_JAVA_OPTS% "-Dactivator.home=%JAVA_FRIENDLY_HOME%" -jar "%ACTIVATOR_HOME%\%ACTIVATOR_LAUNCH_JAR%" %CMDS%
if NOT "%_JAVACMD%"=="%NON_SPACED_CMD%" "%_JAVACMD%" %DEBUG_OPTS% %MEM_OPTS% %ACTIVATOR_OPTS% %SBT_OPTS% %_JAVA_OPTS% "-Dactivator.home=%JAVA_FRIENDLY_HOME%" -jar "%ACTIVATOR_HOME%\%ACTIVATOR_LAUNCH_JAR%" %CMDS%
if ERRORLEVEL 1 goto error
goto end
:error
set ERROR_CODE=1
:end
@endlocal
exit /B %ERROR_CODE%

36
app/Filters.scala Normal file
View File

@@ -0,0 +1,36 @@
import javax.inject.Inject
import play.api._
import play.api.http.HttpFilters
import play.api.libs.iteratee.{Done, Iteratee}
import play.api.mvc._
import play.filters.csrf.CSRFFilter
import play.twirl.api.Txt
import scala.concurrent.Future
class HostnameValidatingAction(allowedHostnames: Set[String], allowAllIps: Boolean, next: EssentialAction) extends EssentialAction with Results{
private val IpAddressPatternComponent = // comes from http://www.mkyong.com/regular-expressions/how-to-validate-ip-address-with-regular-expression/
"([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
"([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
"([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
"([01]?\\d\\d?|2[0-4]\\d|25[0-5])"
private val IpAddress = ("""^"""+IpAddressPatternComponent+"""((:[0-9]+)?)$""").r
override def apply(request: RequestHeader): Iteratee[Array[Byte], Result] = {
if( (allowedHostnames contains request.host) || (allowAllIps && IpAddress.findFirstMatchIn(request.host).isDefined )) next.apply(request)
else Iteratee.flatten(Future.successful(Done(Unauthorized(Txt(s"not allowed for host ${request.host}")))))
}
}
class HostnameFilter(allowedHostnames: Set[String], allowAllIps: Boolean = false) extends EssentialFilter {
override def apply(next: EssentialAction): EssentialAction = new HostnameValidatingAction(allowedHostnames, allowAllIps, next)
}
class Filters @Inject() (csrfFilter: CSRFFilter, configuration: Configuration) extends HttpFilters {
def filters = Seq(csrfFilter, new HostnameFilter(configuration.getString("app.hostname").toSet, allowAllIps = true))
}

123
app/assets/css/main.css Normal file
View File

@@ -0,0 +1,123 @@
#main{
padding-top: 50px;
}
.vuln-details{
border-collapse: collapse;
}
.vuln-details td, .vuln-details th {
padding: 5px;
border: 1px solid gray;
}
.toggle-warning {
float: left;
margin-right: 7px;
}
.versionless-dependency:not(:hover) .related-links{
display: none;
}
.description{
border-left: 3px solid black;
margin-bottom: 3px;
padding-left: 5px;
}
.badge a{
color: yellow;
}
h2:before, h3:before, h4:before, h5:before, h6:before {
display: block;
content: " ";
margin-top: -50px;
height: 50px;
visibility: hidden;
}
#project-selector{
float: left;
padding: 5px;
padding-top: 10px;
}
#project-selector button{
max-width: 200px;
overflow: hidden;
text-overflow: ellipsis;
}
#project-selector .dropdown-menu{
max-height: 300px;
overflow: auto;
}
#project-selector .base-project a{
font-weight: bolder;
}
h3.library-identification{
border: 1px solid black;
padding: 5px;
}
.library-identification-badge-hack{
display: inline-block;
width: 1px;
padding-right: 0;
padding-left: 0;
margin-left: 0;
margin-right: 0;
visibility: hidden;
}
.jqplot-table-legend{
background-color: rgba(255, 255, 255, 0.25);
padding: 5px;
margin-right: 5px;
}
.jqplot-table-legend-swatch{
height: 10px;
width: 10px;
margin-right: 5px;
}
.jqplot-data-label{
color: white;
font-weight: bold;
}
.severity{
font-size: smaller;
margin-bottom: 5px;
}
.severity .score{
font-weight: bolder;
}
.explained{
border-bottom: 1px dashed black;
}
.explained:after{
content: "?";
font-weight: bold;
display: inline-block;
color: white;
background-color: black;
border: 1px solid black;
border-radius: 100%;
text-align: center;
vertical-align: middle;
height: 1.5em;
width: 1.5em;
margin-left: 0.5em;
font-size: 50%;
}
.explained:hover{
border-bottom-color: blue;
}
.explained:hover:after{
background-color: blue;
border-color: blue;
}
.help{
border-left: 5px solid #00c7ff;
padding-left: 10px;
}

59
app/assets/js/main.js Normal file
View File

@@ -0,0 +1,59 @@
function toggleTag(el){
var btn = $(el);
var tagId = parseInt(btn.attr("data-tag-id"));
var libraryId = parseInt(btn.attr("data-library-id"));
btn.attr({disabled: true});
var add = !btn.hasClass("btn-success");
var libraryTagPair = {tagId: tagId, libraryId: libraryId};
$.ajax({
url: add ? Routes.addTag : Routes.removeTag,
method: 'POST',
//dataType: 'json',
data: JSON.stringify(
add ? { libraryTagPair: libraryTagPair, contextDependent: false} : libraryTagPair
),
contentType : 'application/json',
success: function(){
if(add){
btn.addClass("btn-success");
}else{
btn.removeClass("btn-success");
}
btn.attr({disabled: false});
//alert("SUCCESS "+add);
},
error: function(){
alert("FAILED!");
btn.addClass("btn-danger");
// Can't enable the button as we can't be sure about the current state
}/*,
complete: function(a, b){
console.log("complete", a, b);
alert(["complete", a, b]);
}*/
});
}
function toggleClassified(el){
var btn = $(el);
var libraryId = parseInt(btn.attr("data-library-id"));
btn.attr({disabled: true});
var classifiedNewValue = !btn.hasClass("btn-success");
$.ajax({
url: Routes.controllers.Application.setClassified(classifiedNewValue).url,
method: 'POST',
contentType : 'application/json',
data: ""+libraryId,
success: function(){
if(classifiedNewValue){
btn.addClass("btn-success");
}else{
btn.removeClass("btn-success");
}
btn.attr({disabled: false});
},
error: function(){
alert("FAILED!");
btn.addClass("btn-danger");
}
});
}

View File

@@ -0,0 +1,38 @@
package binders
import java.net.URLDecoder.decode
import java.net.URLEncoder.encode
import play.api.mvc.{JavascriptLiteral, PathBindable, QueryStringBindable}
object QueryBinders {
/*private def bindableSet[T](implicit seqBinder: QueryStringBindable[Seq[T]]): QueryStringBindable[Set[T]] = seqBinder.transform(
_.toSet,
_.toSeq
)
implicit def bindableSetOfInt(implicit seqBinder: QueryStringBindable[Seq[Int]]): QueryStringBindable[Set[Int]] = bindableSet[Int]*/
import play.api.libs.json._
private val formats = implicitly[Format[Map[String, Int]]]
implicit def bindableMapStringToInt: QueryStringBindable[Map[String, Int]] = {
QueryStringBindable.bindableString.transform(s => formats.reads(Json.parse(s)).getOrElse(Map()), map => formats.writes(map).toString())
}
implicit object MapStringIntJavascriptLiteral extends JavascriptLiteral[Map[String, Int]] {
override def to(value: Map[String, Int]): String = formats.writes(value).toString()
}
implicit val StringOptionPathBindable: PathBindable[Option[String]] = implicitly[PathBindable[String]].transform(
{
case "" => None
case x => Some(decode(x, "utf-8"))
},
_.map(encode(_, "utf-8")).getOrElse("")
)
//implicit def somePathBindable[T : PathBindable]: PathBindable[Some[T]] = implicitly[PathBindable[T]].transform(Some(_), _.x)
}

View File

@@ -0,0 +1,166 @@
package com.ysoft.odc
import com.google.inject.Inject
import com.google.inject.name.Named
import org.ccil.cowan.tagsoup.jaxp.SAXFactoryImpl
import play.api.libs.ws.{WS, WSAuthScheme, WSClient, WSRequest}
import upickle.default._
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}
import scala.xml.Node
final case class Link(
href: String,
rel: String
)
final case class Artifact(
name: String,
link: Link
//size: Option[Long]
){
def url: String = link.href
}
final case class Artifacts(
size: Int,
//`start-index`: Int,
//`max-result`: Int
artifact: Seq[Artifact]
)
final case class Build(
state: String,
//link: Link,
buildResultKey: String,
buildState: String,
projectName: String,
artifacts: Artifacts
) {
def resultLink(urlBase: String): String = s"$urlBase/browse/$buildResultKey/log"
}
sealed trait FlatArtifactItem{
def name: String
}
abstract sealed class ArtifactItem{
def name: String
final def flatFiles: Map[String, Array[Byte]] = flatFilesWithPrefix("")
def flatFilesWithPrefix(prefix: String): Map[String, Array[Byte]]
def toTree(indent: Int = 0): String
def toTree: String = toTree(0)
}
final case class ArtifactFile(name: String, data: Array[Byte]) extends ArtifactItem with FlatArtifactItem{
override def toTree(indent: Int): String = " "*indent + s"$name = $data"
override def flatFilesWithPrefix(prefix: String): Map[String, Array[Byte]] = Map(prefix + name -> data)
def dataString = new String(data, "utf-8")
}
final case class ArtifactDirectory(name: String, items: Map[String, ArtifactItem]) extends ArtifactItem{
override def toTree(indent: Int): String = " "*indent + s"$name:\n"+items.values.map(_.toTree(indent+2)).mkString("\n")
override def flatFilesWithPrefix(prefix: String): Map[String, Array[Byte]] = items.values.flatMap(_.flatFilesWithPrefix(s"$prefix$name/")).toMap
}
final case class FlatArtifactDirectory(name: String, items: Seq[(String, String)]) extends FlatArtifactItem{}
trait BambooAuthentication{
def addAuth(request: WSRequest): WSRequest
}
class SessionIdBambooAuthentication(sessionId: String) extends BambooAuthentication{
override def addAuth(request: WSRequest): WSRequest = request.withHeaders("Cookie" -> s"JSESSIONID=${sessionId.takeWhile(_.isLetterOrDigit)}")
}
class CredentialsBambooAuthentication(user: String, password: String) extends BambooAuthentication{
override def addAuth(request: WSRequest): WSRequest = request.withQueryString("os_authType" -> "basic").withAuth(user, password, WSAuthScheme.BASIC)
}
final class BambooDownloader @Inject() (@Named("bamboo-server-url") val server: String, auth: BambooAuthentication)(implicit executionContext: ExecutionContext, wSClient: WSClient) extends Downloader {
private object ArtifactKeys{
val BuildLog = "Build log"
val ResultsHtml = "Report results-HTML"
val ResultsXml = "Report results-XML"
}
private def downloadArtifact(artifactMap: Map[String, Artifact], key: String)(implicit wSClient: WSClient): Future[FlatArtifactItem] = {
val artifact = artifactMap(key)
downloadArtifact(artifact.url, artifact.name)
}
private def downloadArtifact(url: String, name: String)(implicit wSClient: WSClient): Future[FlatArtifactItem] = {
bambooUrl(url).get().map{response =>
response.header("Content-Disposition") match{
case Some(_) => ArtifactFile(name = name, data = response.bodyAsBytes)
case None =>
val html = response.body
val hpf = new SAXFactoryImpl
hpf.setFeature("http://xml.org/sax/features/external-general-entities", false)
//hpf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true)
hpf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
val HtmlParser = hpf.newSAXParser()
val Html = scala.xml.XML.withSAXParser(HtmlParser)
val xml = Html.loadString(html)
val tds = xml \\ "td"
val subdirs = tds flatMap { td =>
(td \ "img").headOption.flatMap{img =>
val suffix = img.attribute("alt").map(_.text) match { // suffix seems to be no longer needed, as we recognize directories elsehow
case Some("(dir)") => "/"
case Some("(file)") => ""
case other => sys.error(s"unexpected directory item type: $other")
}
(td \ "a").headOption.map{ link =>
val hrefAttribute: Option[Seq[Node]] = link.attribute("href")
link.text -> (hrefAttribute.getOrElse(sys.error(s"bad link $link at $url")).text+suffix) : (String, String)
} : Option[(String, String)]
} : Option[(String, String)]
}
FlatArtifactDirectory(name = name, items = subdirs)
}
}
}
private def downloadArtifactRecursively(artifactMap: Map[String, Artifact], key: String)(implicit wSClient: WSClient): Future[ArtifactItem] = {
val artifact = artifactMap(key)
downloadArtifactRecursively(url = artifact.url, name = artifact.name)
}
private def downloadArtifactRecursively(url: String, name: String/*artifactMap: Map[String, Artifact], key: String*/)(implicit wSClient: WSClient): Future[ArtifactItem] = {
downloadArtifact(url/*artifactMap, key*/, name).flatMap{
case directoryStructure: FlatArtifactDirectory =>
Future.traverse(directoryStructure.items){case (subName, urlString) =>
downloadArtifactRecursively(server+urlString, subName)
}.map{ items =>
ArtifactDirectory(name = directoryStructure.name, items = items.map(i => i.name->i).toMap)
}
case file: ArtifactFile => Future.successful(file)
}
}
override def downloadProjectReports(projects: Set[String], requiredVersions: Map[String, Int]): Future[(Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable])] = {
val resultsFuture = Future.traverse(projects){project =>
downloadProjectReport(project, requiredVersions.get(project))
}
resultsFuture.map{ results =>
val (successfulReportTries, failedReportTries) = results.partition(_._2.isSuccess)
val successfulReports = successfulReportTries.map{case (name, Success(data)) => name -> data; case _ => ???}.toMap
val failedReports = failedReportTries.map{case (name, Failure(data)) => name -> data; case _ => ???}.toMap
(successfulReports, failedReports)
}
}
private def bambooUrl(url: String) = auth.addAuth(WS.clientUrl(url))
private def downloadProjectReport(project: String, versionOption: Option[Int]): Future[(String, Try[(Build, ArtifactItem, ArtifactFile)])] = {
val url = s"$server/rest/api/latest/result/$project-${versionOption.getOrElse("latest")}.json?expand=artifacts"
val resultFuture = (bambooUrl(url).get().flatMap { response =>
val build = read[Build](response.body)
val artifactMap: Map[String, Artifact] = build.artifacts.artifact.map(x => x.name -> x).toMap
val logFuture = downloadArtifact(artifactMap, ArtifactKeys.BuildLog).map(_.asInstanceOf[ArtifactFile])
val reportsFuture: Future[ArtifactItem] = downloadArtifactRecursively(artifactMap, ArtifactKeys.ResultsXml)
for {
log <- logFuture
reports <- reportsFuture
} yield (build, reports, log)
}: Future[(Build, ArtifactItem, ArtifactFile)])
resultFuture.map(data => project -> Success(data)).recover{case e => project -> Failure(e)}
}
}

View File

@@ -0,0 +1,33 @@
package com.ysoft.odc
import controllers.WarningSeverity.WarningSeverity
import controllers.{IdentifiedWarning, ReportInfo, Warning}
import play.twirl.api.{Html, HtmlFormat}
object Checks {
def differentValues(id: String, name: String, severity: WarningSeverity)(f: Map[ReportInfo, Analysis] => Traversable[_]) = { (data: Map[ReportInfo, Analysis]) =>
val variants = f(data)
if(variants.size > 1){
Some(IdentifiedWarning(id, HtmlFormat.escape(s"different $name!"), severity))
}else{
None
}
}
def badValues(id: String, name: String, severity: WarningSeverity)(f: (ReportInfo, Analysis) => Option[Html]): Map[ReportInfo, Analysis] => Option[Warning] = { (data: Map[ReportInfo, Analysis]) =>
val badValues = data.collect(Function.unlift{case (analysisName, analysis) => f(analysisName, analysis).map(analysisName -> _)}).toSeq
if(badValues.size > 0) Some(IdentifiedWarning(id, views.html.warnings.badValues(name, badValues), severity))
else None
}
def badGroupedDependencies[C <: Traversable[_]](id: String, name: String, severity: WarningSeverity)(f: Seq[GroupedDependency] => C)(show: C => Traversable[_] = {(x: C) => x}, exclusions: Set[Exclusion] = Set()): (Seq[GroupedDependency] => Option[Warning]) = { (data: Seq[GroupedDependency]) =>
val badItems = f(data.filterNot(ds => exclusions.exists(_.matches(ds))))
if(badItems.size > 0){
Some(IdentifiedWarning(id, views.html.warnings.badGroupedDependencies(name, badItems.size, show(badItems)), severity))
}else{
None
}
}
}

View File

@@ -0,0 +1,10 @@
package com.ysoft.odc
import scala.concurrent.Future
/**
* Created by user on 10/30/15.
*/
trait Downloader {
def downloadProjectReports(projects: Set[String], requiredVersions: Map[String, Int]): Future[(Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable])]
}

View File

@@ -0,0 +1,17 @@
package com.ysoft.odc
import javax.inject.Named
import com.google.inject.Inject
import scala.concurrent.Future
class LocalFilesDownloader @Inject() (@Named("reports-path") path: String) extends Downloader{
override def downloadProjectReports(projects: Set[String], requiredVersions: Map[String, Int]): Future[(Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable])] = {
if(requiredVersions != Map()){
sys.error("Versions are not supported there")
}
projects.map{pn => ???}
???
}
}

View File

@@ -0,0 +1,293 @@
package com.ysoft.odc
import com.github.nscala_time.time.Imports._
import controllers.ReportInfo
import models.{LibraryType, PlainLibraryIdentifier}
import scala.xml._
final case class SerializableXml private (xmlString: String, @transient private val xmlData: NodeSeq) extends Serializable{
@transient lazy val xml = Option(xmlData).getOrElse(SecureXml.loadString(xmlString))
override def equals(obj: scala.Any): Boolean = obj match {
case SerializableXml(s, _) => s == this.xmlString
}
override def hashCode(): Int = 42+xmlString.hashCode
}
object SerializableXml{
def apply(xml: Node): SerializableXml = SerializableXml(xml.toString(), xml)
def apply(xml: NodeSeq): SerializableXml = SerializableXml(xml.toString(), xml)
}
final case class Analysis(scanInfo: SerializableXml, name: String, reportDate: DateTime, dependencies: Seq[Dependency])
final case class Hashes(sha1: String, md5: String){
override def toString: String = s"Hashes(sha1=$sha1, md5=$md5)"
}
final case class Exclusion(sha1: String) extends AnyVal {
def matches(dependency: Dependency): Boolean = dependency.sha1 == sha1
def matches(group: GroupedDependency): Boolean = group.sha1 == sha1
}
final case class Evidence(source: String, name: String, value: String, confidence: String, evidenceType: String)
final case class Dependency(
fileName: String,
filePath: String,
md5: String,
sha1: String,
description: String,
evidenceCollected: Set[Evidence],
identifiers: Seq[Identifier],
license: String,
vulnerabilities: Seq[Vulnerability],
suppressedVulnerabilities: Seq[Vulnerability],
relatedDependencies: SerializableXml
){
def hashes = Hashes(sha1 = sha1, md5 = md5)
def plainLibraryIdentifiers: Set[PlainLibraryIdentifier] = identifiers.flatMap(_.toLibraryIdentifierOption).toSet
/*
Method equals seems to be a CPU hog there. I am not sure if we can do something reasonable about it.
We can compare by this.hashes, but, in such case, dependencies that differ in evidence will be considered the same if their JAR hashes are the same, which would break some sanity checks.
*/
}
/**
* A group of dependencies having the same fingerprints
* @param dependencies
*/
final case class GroupedDependency(dependencies: Map[Dependency, Set[ReportInfo]]) {
def parsedDescriptions: Seq[Seq[Seq[String]]] = descriptions.toSeq.sorted.map(_.split("\n\n").toSeq.map(_.split("\n").toSeq))
def isVulnerable: Boolean = vulnerabilities.nonEmpty
def maxCvssScore = (Seq(None) ++ vulnerabilities.map(_.cvssScore)).max
def ysdssScore = maxCvssScore.map(_ * projects.size)
def descriptions = dependencies.keySet.map(_.description)
def projects = dependencies.values.flatten.toSet
def fileNames = dependencies.keySet.map(_.fileName)
def hashes = dependencies.keys.head.hashes // valid since all deps in a group have the same hashes
val sha1 = hashes.sha1
def identifiers: Set[Identifier] = dependencies.keySet.flatMap(_.identifiers)
def mavenIdentifiers = identifiers.filter(_.identifierType == "maven")
def cpeIdentifiers = identifiers.filter(_.identifierType == "cpe")
def vulnerabilities: Set[Vulnerability] = dependencies.keySet.flatMap(_.vulnerabilities)
def plainLibraryIdentifiers: Set[PlainLibraryIdentifier] = identifiers.flatMap(_.toLibraryIdentifierOption)
def hasCpe: Boolean = cpeIdentifiers.nonEmpty
}
object GroupedDependency{
def apply(deps: Seq[(Dependency, ReportInfo)]): GroupedDependency = GroupedDependency(deps.groupBy(_._1).mapValues(_.map(_._2).toSet)) // TODO: the groupBy seems to be a CPU hog (because of GroupedDependency.equals); The mapValues is lazy, so its repeated might also be a performance hog, but I doubt that values are used frequently.
}
object Confidence extends Enumeration {
type Confidence = Value
// Order is important
val Low = Value("LOW")
val Medium = Value("MEDIUM")
val High = Value("HIGH")
val Highest = Value("HIGHEST")
}
final case class Reference(source: String, url: String, name: String)
final case class VulnerableSoftware(allPreviousVersion: Boolean, name: String)
final case class CvssRating(score: Option[Double], authenticationr: Option[String], availabilityImpact: Option[String], accessVector: Option[String], integrityImpact: Option[String], accessComplexity: Option[String], confidentialImpact: Option[String])
final case class CWE(name: String) extends AnyVal{
override def toString = name
def brief = name.takeWhile(_ != ' ')
def numberOption: Option[Int] = if(brief startsWith "CWE-") try {
Some(brief.substring(4).toInt)
} catch {
case _: NumberFormatException => None
} else None
}
final case class Vulnerability(name: String, cweOption: Option[CWE], cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){
def cvssScore = cvss.score
def ysvssScore(affectedDeps: Set[GroupedDependency]) = cvssScore.map(_ * affectedDeps.flatMap(_.projects).toSet.size)
}
final case class Identifier(name: String, confidence: Confidence.Confidence, url: String, identifierType: String) {
def toLibraryIdentifierOption: Option[PlainLibraryIdentifier] = {
if(identifierType == "maven"){
val groupId::artifactId::_ = name.split(':').toList
Some(PlainLibraryIdentifier(libraryType = LibraryType.Maven, libraryIdentifier = s"$groupId:$artifactId"))
}else{
None
}
}
def toCpeIdentifierOption: Option[String] = identifierType match {
case "cpe" => Some(name)
case _ => None
}
//def isClassifiedInSet(set: Set[PlainLibraryIdentifier]): Boolean = toLibraryIdentifierOption.exists(set contains _)
}
object OdcParser {
def filterWhitespace(node: Node) = node.nonEmptyChildren.filter{
case t: scala.xml.Text if t.text.trim == "" => false
case t: scala.xml.PCData if t.text.trim == "" => false
case _ => true
}
def checkElements(node: Node, knownElements: Set[String]) {
val subelementNames = filterWhitespace(node).map(_.label).toSet
val unknownElements = subelementNames -- knownElements
if(unknownElements.nonEmpty){
sys.error("Unknown elements for "+node.label+": "+unknownElements)
}
}
private def getAttributes(data: MetaData): List[String] = data match {
case Null => Nil
case Attribute(key, _, next) => key :: getAttributes(next)
}
def checkParams(node: Node, knownParams: Set[String]) {
val paramNames = getAttributes(node.attributes).toSet
val unknownParams = paramNames -- knownParams
if(unknownParams.nonEmpty){
sys.error("Unknown params for "+node.label+": "+unknownParams)
}
}
def parseVulnerableSoftware(node: Node): VulnerableSoftware = {
checkElements(node, Set("#PCDATA"))
checkParams(node, Set("allPreviousVersion"))
if(node.label != "software"){
sys.error(s"Unexpected element for vulnerableSoftware: ${node.label}")
}
VulnerableSoftware(
name = node.text,
allPreviousVersion = node.attribute("allPreviousVersion").map(_.text).map(Map("true"->true, "false"->false)).getOrElse(false)
)
}
def parseReference(node: Node): Reference = {
checkElements(node, Set("source", "url", "name"))
checkParams(node, Set())
if(node.label != "reference"){
sys.error(s"Unexpected element for reference: ${node.label}")
}
Reference(
source = (node \ "source").text,
url = (node \ "url").text,
name = (node \ "name").text
)
}
def parseVulnerability(node: Node, expectedLabel: String = "vulnerability"): Vulnerability = {
checkElements(node, Set("name", "severity", "cwe", "cvssScore", "description", "references", "vulnerableSoftware", "cvssAuthenticationr", "cvssAvailabilityImpact", "cvssAccessVector", "cvssIntegrityImpact", "cvssAccessComplexity", "cvssConfidentialImpact"))
if(node.label != expectedLabel){
sys.error(s"Unexpected element for vuln: ${node.label}")
}
def t(ns: NodeSeq) = {
ns match {
case Seq() => None
case Seq(one) =>
one.attributes match {
case Null =>
one.child match {
case Seq(hopefullyTextChild) =>
hopefullyTextChild match {
case Text(data) => Some(data)
}
}
}
}
}
Vulnerability(
name = (node \ "name").text,
//severity = (node \ "severity"), <- severity is useless, as it is computed from cvssScore :D
cweOption = (node \ "cwe").headOption.map(_.text).map(CWE),
description = (node \ "description").text,
cvss = CvssRating(
score = (node \ "cvssScore").headOption.map(_.text.toDouble),
authenticationr = t(node \ "cvssAuthenticationr"),
availabilityImpact = t(node \ "cvssAvailabilityImpact"),
accessVector = t(node \ "cvssAccessVector"),
integrityImpact = t(node \ "cvssIntegrityImpact"),
accessComplexity = t(node \ "cvssAccessComplexity"),
confidentialImpact = t(node \ "cvssConfidentialImpact")
),
references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_)),
vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware)
)
}
def parseIdentifier(node: Node): Identifier = {
checkElements(node, Set("name", "url"))
checkParams(node, Set("type", "confidence"))
val ExtractPattern = """\((.*)\)""".r
Identifier(
name = (node \ "name").text match {
case ExtractPattern(text) => text
},
url = (node \ "url").text,
identifierType = node.attribute("type").get.text,
confidence = Confidence.withName(node.attribute("confidence").get.text)
)
}
def parseIdentifiers(seq: Node): Seq[Identifier] = {
filterWhitespace(seq.head).map(parseIdentifier(_))
}
def parseDependency(node: Node): Dependency = {
checkElements(node, Set("fileName", "filePath", "md5", "sha1", "description", "evidenceCollected", "identifiers", "license", "vulnerabilities", "relatedDependencies"))
checkParams(node, Set())
val (vulnerabilities: Seq[Node], suppressedVulnerabilities: Seq[Node]) = (node \ "vulnerabilities").headOption.map(filterWhitespace).getOrElse(Seq()).partition(_.label == "vulnerability")
Dependency(
fileName = (node \ "fileName").text,
filePath = (node \ "filePath").text,
md5 = (node \ "md5").text,
sha1 = (node \ "sha1").text,
description = (node \ "description").text,
evidenceCollected = filterWhitespace((node \ "evidenceCollected").head).map(parseEvidence).toSet,
identifiers = (node \ "identifiers").headOption.map(parseIdentifiers).getOrElse(Seq()),
license = (node \ "license").text,
vulnerabilities = vulnerabilities.map(parseVulnerability(_)),
suppressedVulnerabilities = suppressedVulnerabilities.map(parseVulnerability(_, "suppressedVulnerability")),
relatedDependencies = SerializableXml(node \ "relatedDependencies")
)
}
def parseEvidence(node: Node): Evidence = {
if(node.label != "evidence"){
sys.error(s"Unexpected element for evidence: ${node.label}")
}
checkElements(node, Set("source", "name", "value"))
checkParams(node, Set("confidence", "type"))
Evidence(
source = (node \ "source").text,
name = (node \ "name").text,
value = (node \ "value").text,
confidence = node.attribute("confidence").map(_.text).get,
evidenceType = node.attribute("type").map(_.text).get
)
}
def parseDependencies(nodes: NodeSeq): Seq[Dependency] = nodes.map(parseDependency(_))
def parseXmlReport(data: Array[Byte]) = {
val xml = SecureXml.loadString(new String(data, "utf-8"))
Analysis(
scanInfo = SerializableXml((xml \ "scanInfo").head),
name = (xml \ "projectInfo" \ "name").text,
reportDate = DateTime.parse((xml \ "projectInfo" \ "reportDate").text),
dependencies = parseDependencies(xml \ "dependencies" \ "dependency").toIndexedSeq
)
}
}

View File

@@ -0,0 +1,17 @@
package com.ysoft.odc
import javax.xml.parsers.SAXParserFactory
import scala.xml.{Elem, XML}
// copied from https://github.com/scala/scala-xml/issues/17 and slightly modified
object SecureXml {
def loadString(xml: String): Elem = {
val spf = SAXParserFactory.newInstance()
spf.setFeature("http://xml.org/sax/features/external-general-entities", false)
spf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true)
val saxParser = spf.newSAXParser()
XML.withSAXParser(saxParser).loadString(xml)
}
}

View File

@@ -0,0 +1,293 @@
package controllers
import java.sql.BatchUpdateException
import com.github.nscala_time.time.Imports._
import com.google.inject.Inject
import com.google.inject.name.Named
import models._
import play.api.Logger
import play.api.data.Forms._
import play.api.data._
import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider}
import play.api.http.ContentTypes
import play.api.i18n.{I18nSupport, MessagesApi}
import play.api.libs.json._
import play.api.mvc._
import play.api.routing.JavaScriptReverseRouter
import play.twirl.api.Txt
import services.{LibrariesService, LibraryTagAssignmentsService, TagsService}
import views.html.DefaultRequest
import scala.collection.immutable.SortedMap
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
object ApplicationFormats{
implicit val libraryTagPairFormat = Json.format[LibraryTagPair]
implicit val libraryTagAssignmentFormat = Json.format[LibraryTagAssignment]
//implicit val libraryTypeFormat = Json.format[LibraryType]
//implicit val plainLibraryIdentifierFormat = Json.format[PlainLibraryIdentifier]
//implicit val libraryFormat = Json.format[Library]
implicit val libraryTagFormat = Json.format[LibraryTag]
}
object export {
import ApplicationFormats._
final case class AssignedTag(name: String, contextDependent: Boolean)
final case class TaggedLibrary(identifier: String, classified: Boolean, tags: Seq[AssignedTag]){
def toLibrary = Library(plainLibraryIdentifier = PlainLibraryIdentifier.fromString(identifier), classified = classified)
}
final case class Export(libraryMapping: Seq[TaggedLibrary], tags: Seq[LibraryTag])
implicit val assignedTagFormats = Json.format[AssignedTag]
implicit val taggedLibraryFormats = Json.format[TaggedLibrary]
implicit val exportFormats = Json.format[Export]
}
class Application @Inject() (
reportsParser: DependencyCheckReportsParser,
reportsProcessor: DependencyCheckReportsProcessor,
projectReportsProvider: ProjectReportsProvider,
@Named("missing-GAV-exclusions") missingGAVExclusions: MissingGavExclusions,
tagsService: TagsService,
librariesService: LibrariesService,
libraryTagAssignmentsService: LibraryTagAssignmentsService,
protected val dbConfigProvider: DatabaseConfigProvider,
val messagesApi: MessagesApi,
val env: AuthEnv
) extends AuthenticatedController with HasDatabaseConfigProvider[models.profile.type]{
import ApplicationFormats._
import dbConfig.driver.api._
import models.tables.snoozesTable
import reportsProcessor.processResults
import secureRequestConversion._
val dateFormatter = DateTimeFormat.forPattern("dd-MM-yyyy")
val emptySnoozeForm = Form(mapping(
"until" -> text.transform(LocalDate.parse(_, dateFormatter), (_: LocalDate).toString(dateFormatter)).verifying("Must be a date in the future", _ > LocalDate.now),
//"snoozed_object_identifier" -> text,
"reason" -> text(minLength = 3, maxLength = 255)
)(ObjectSnooze.apply)(ObjectSnooze.unapply))
def loadSnoozes() = {
val now = LocalDate.now
import models.jodaSupport._
for{
bareSnoozes <- db.run(snoozesTable.filter(_.until > now).result) : Future[Seq[(Int, Snooze)]]
snoozes = bareSnoozes.groupBy(_._2.snoozedObjectId).mapValues(ss => SnoozeInfo(emptySnoozeForm, ss.sortBy(_._2.until))).map(identity)
} yield snoozes.withDefaultValue(SnoozeInfo(emptySnoozeForm, Seq()))
}
def purgeCache(versions: Map[String, Int], next: String) = Action {
projectReportsProvider.purgeCache(versions)
next match {
case "index" => Redirect(routes.Application.index(versions))
case _ => Ok(Txt("CACHE PURGED"))
}
}
def index(versions: Map[String, Int]) = ReadAction.async{ implicit req =>
loadSnoozes() flatMap { snoozes =>
indexPage(versions)(snoozes, securedRequestToUserAwareRequest(req))
}
}
def indexPage(requiredVersions: Map[String, Int])(implicit snoozes: SnoozesInfo, requestHeader: DefaultRequest) = {
val (lastRefreshTimeFuture, resultsFuture) = projectReportsProvider.resultsForVersions(requiredVersions)
processResults(resultsFuture, requiredVersions).flatMap{ case (vulnerableDependencies, allWarnings, groupedDependencies) =>
Logger.debug("indexPage: Got results")
//val unclassifiedDependencies = groupedDependencies.filterNot(ds => MissingGAVExclusions.exists(_.matches(ds))).filterNot(_.identifiers.exists(_.isClassifiedInSet(classifiedSet)))
for{
knownDependencies <- librariesService.allBase
_ = Logger.debug("indexPage: #1")
includedDependencies = groupedDependencies.filterNot(missingGAVExclusions.isExcluded)
_ = Logger.debug("indexPage: #2")
unknownDependencies = includedDependencies.flatMap(_.identifiers.flatMap(_.toLibraryIdentifierOption)).toSet -- knownDependencies.map(_.plainLibraryIdentifier).toSet
_ = Logger.debug("indexPage: #3")
_ <- librariesService.insertMany(unknownDependencies.map(Library(_, classified = false)))
_ = Logger.debug("indexPage: #3")
unclassifiedDependencies <- librariesService.unclassified
_ = Logger.debug("indexPage: #4")
allTags <- tagsService.all
_ = Logger.debug("indexPage: #6")
allTagsMap = allTags.toMap
_ = Logger.debug("indexPage: #7")
tagsWithWarning = allTags.collect(Function.unlift{case (id, t: LibraryTag) => t.warningOrder.map(_ => (id, t))}).sortBy(_._2.warningOrder)
_ = Logger.debug("indexPage: #8")
librariesForTagsWithWarningUnsorted <- librariesService.librariesForTags(tagsWithWarning.map(_._1))
_ = Logger.debug("indexPage: #9")
librariesForTagsWithWarning = SortedMap(librariesForTagsWithWarningUnsorted.groupBy(_._1).toSeq.map{case (tagId, lr) => (tagId, allTagsMap(tagId)) -> lr.map(_._2) } : _*)(Ordering.by(t => (t._2.warningOrder, t._1)))
_ = Logger.debug("indexPage: #10")
relatedDependenciesTags <- librariesService.byTags(unclassifiedDependencies.map(_._1).toSet ++ librariesForTagsWithWarning.values.flatten.map(_._1).toSet)
_ = Logger.debug("indexPage: #11")
lastRefreshTime <- lastRefreshTimeFuture
} yield {
Logger.debug("indexPage: Got all ingredients")
Ok(views.html.index(
vulnerableDependencies = vulnerableDependencies,
warnings = allWarnings,
librariesForTagsWithWarning = librariesForTagsWithWarning,
unclassifiedDependencies = unclassifiedDependencies,
groupedDependencies = groupedDependencies,
dependenciesForLibraries = groupedDependencies.flatMap(group =>
group.identifiers.flatMap(_.toLibraryIdentifierOption).map(_ -> group)
).groupBy(_._1).mapValues(_.map(_._2).toSet).map(identity),
allTags = allTags,
relatedDependenciesTags = relatedDependenciesTags,
lastRefreshTime = lastRefreshTime,
versions = requiredVersions
))
}
} recover {
case e: BatchUpdateException =>
throw e.getNextException
}
}
implicit class AddAdjustToMap[K, V](m: Map[K, V]){
def adjust(k: K)(f: V => V) = m.updated(k, f(m(k)))
}
def snooze(id: String, versions: Map[String, Int]) = AdminAction.async { implicit req =>
loadSnoozes().flatMap{ loadedSnoozes =>
val snoozes = loadedSnoozes.adjust(id){_.adjustForm(_.bindFromRequest()(req))}
snoozes(id).form.fold(
f => indexPage(Map())(snoozes, securedRequestToUserAwareRequest(req)),
snooze => for {
_ <- db.run(snoozesTable.map(_.base) += snooze.toSnooze(id))
} yield Redirect(routes.Application.index(versions).withFragment(id))
)
}
}
def unsnooze(snoozeId: Int, versions: Map[String, Int]) = AdminAction.async { implicit req =>
(db.run(snoozesTable.filter(_.id === snoozeId).map(_.base).result).map(_.headOption): Future[Option[Snooze]]).flatMap {
case Some(snooze) =>
for(_ <- db.run(snoozesTable.filter(_.id === snoozeId).delete)) yield Redirect(routes.Application.index(versions).withFragment(snooze.snoozedObjectId))
case None => Future.successful(NotFound(Txt("Unknown snoozeId")))
}
}
// TODO: move import/export to a separate controller
def tagsExport = Action.async {
import export._
for{
tags <- tagsService.all.map(_.toMap)
lta <- libraryTagAssignmentsService.byLibrary
libs <- librariesService.touched(lta.keySet)
} yield {
val libraryMapping = (libs: Seq[(Int, Library)]).sortBy(_._2.plainLibraryIdentifier.toString).map { case (id, l) =>
val assignments: Seq[LibraryTagAssignment] = lta(id)
TaggedLibrary(
identifier = s"${l.plainLibraryIdentifier}",
classified = l.classified,
tags = assignments.map(a => AssignedTag(name = tags(a.tagId).name, contextDependent = a.contextDependent)).sortBy(_.name.toLowerCase)
)
}
Ok(Json.prettyPrint(Json.toJson(
Export(libraryMapping = libraryMapping, tags = tags.values.toSeq.sortBy(_.name.toLowerCase))
))).as(ContentTypes.JSON)
}
}
val tagsImportForm = Form(mapping("data" -> text)(identity)(Some(_)))
def tagsImport = AdminAction { implicit req =>
Ok(views.html.tagsImport(tagsImportForm))
}
def tagsImportAction = AdminAction.async { implicit req =>
tagsImportForm.bindFromRequest()(req).fold(
formWithErrors => ???,
data =>
export.exportFormats.reads(Json.parse(data)).fold(
invalid => Future.successful(BadRequest(Txt("ERROR: "+invalid))),
data => {
def importTags() = tagsService.insertMany(data.tags)
def getTagsByName(): Future[Map[String, Int]] = tagsService.all.map(_.groupBy(_._2.name).mapValues { case Seq((id, _)) => id }.map(identity))
def importLibraries(): Future[Unit] = Future.sequence(
data.libraryMapping.map{ taggedLibrary =>
librariesService.insert(taggedLibrary.toLibrary).flatMap{ libraryId =>
importLibraryTagAssignment(libraryId, taggedLibrary)
}
}
).map( (x: Seq[Unit]) => ()) // I don't care about the result
def importLibraryTagAssignment(libraryId: Int, taggedLibrary: export.TaggedLibrary): Future[Unit] = getTagsByName().flatMap { tagIdsByName =>
Future.sequence(taggedLibrary.tags.map{ assignedTag =>
val tagId = tagIdsByName(assignedTag.name)
libraryTagAssignmentsService.insert(LibraryTagAssignment(LibraryTagPair(libraryId = libraryId, tagId = tagId), assignedTag.contextDependent)).map(_ => ())
}).map( (x: Seq[Unit]) => ()) // I don't care about the result
}
for {
_ <- importTags()
_ <- importLibraries()
} yield Ok(Txt("OK"))
}
)
)
}
def dependencies(requiredClassification: Option[Boolean], requiredTags: Seq[Int], noTag: Boolean) = ReadAction.async { implicit request =>
val requiredTagsSet = requiredTags.toSet
for{
selectedDependencies <- db.run(librariesService.filtered(requiredClassification = requiredClassification, requiredTagsOption = if(noTag) None else Some(requiredTagsSet)).result)
dependencyTags <- librariesService.byTags(selectedDependencies.map(_._1).toSet)
allTags <- tagsService.all
}yield{
Ok(views.html.dependencies(
requiredClassification = requiredClassification,
selectedDependencies = selectedDependencies,
allTags = allTags,
dependencyTags = dependencyTags,
requiredTagSet = requiredTagsSet,
noTag = noTag,
tagsLink = (newTags: Set[Int]) => routes.Application.dependencies(requiredClassification, newTags.toSeq.sorted, noTag),
noTagLink = newNoTag => routes.Application.dependencies(requiredClassification, requiredTagsSet.toSeq.sorted, newNoTag),
classificationLink = newClassification => routes.Application.dependencies(newClassification, requiredTagsSet.toSeq.sorted, noTag)
))
}
}
def removeTag() = AdminAction.async(BodyParsers.parse.json) { request =>
request.body.validate[LibraryTagPair].fold(
err => Future.successful(BadRequest(Txt(err.toString()))),
libraryTagPair => for(_ <- libraryTagAssignmentsService.remove(libraryTagPair)) yield Ok(Txt("OK"))
)
}
def addTag() = AdminAction.async(BodyParsers.parse.json) { request =>
request.body.validate[LibraryTagAssignment].fold(
err => Future.successful(BadRequest(Txt(err.toString()))),
tagAssignment => for(_ <- libraryTagAssignmentsService.insert(tagAssignment)) yield {Ok(Txt("OK"))}
)
}
def setClassified(classified: Boolean) = AdminAction.async(BodyParsers.parse.json) {request =>
val libraryId = request.body.as[Int]
for(_ <- librariesService.setClassified(libraryId, classified)) yield Ok(Txt("OK"))
}
def javascriptRoutes = Action { implicit request =>
Ok(
JavaScriptReverseRouter("Routes")(
routes.javascript.Application.setClassified,
routes.javascript.Application.addTag
)
).as("text/javascript")
}
def testHttps(allowRedirect: Boolean) = Action { Ok(Txt(if(allowRedirect)
"""
|(function(){
| var newUrl = window.location.href.replace(/^http:/, "https:");
| if(newUrl != window.location.href){
| window.location.replace(newUrl);
| }
|})();
|""".stripMargin else "")).withHeaders("Content-type" -> "text/javascript; charset=utf-8") }
}

View File

@@ -0,0 +1,64 @@
package controllers
import javax.inject.Inject
import _root_.services.CredentialsVerificationService
import com.mohiva.play.silhouette.api._
import com.mohiva.play.silhouette.api.util.Clock
import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator
import models.User
import play.api.data.Form
import play.api.data.Forms._
import play.api.i18n.{Messages, MessagesApi}
import play.api.libs.concurrent.Execution.Implicits._
import scala.concurrent.Future
final case class LoginRequest(username: String, password: String, rememberMe: Boolean)
class AuthController @Inject() (
val messagesApi: MessagesApi,
val env: Environment[User, CookieAuthenticator],
clock: Clock,
credentialsVerificationService: CredentialsVerificationService
) extends AuthenticatedController {
val signInForm = Form(mapping(
"username" -> nonEmptyText,
"password" -> nonEmptyText,
"rememberMe" -> boolean
)(LoginRequest.apply)(LoginRequest.unapply))
def signIn = UserAwareAction { implicit request =>
request.identity match {
case Some(user) => Redirect(routes.Application.index(Map()))
case None => Ok(views.html.auth.signIn(signInForm/*, socialProviderRegistry*/))
}
}
def authenticate() = UserAwareAction.async { implicit request =>
signInForm.bindFromRequest().fold(
formWithErrors => Future.successful(BadRequest(views.html.auth.signIn(formWithErrors/*, socialProviderRegistry*/))),
loginRequest => {
credentialsVerificationService.verifyCredentials(loginRequest.username, loginRequest.password).flatMap{
case true =>
val loginInfo: LoginInfo = LoginInfo(providerID = "credentials-verification", providerKey = loginRequest.username)
val user: User = User(username = loginRequest.username)
env.authenticatorService.create(loginInfo) flatMap { authenticator =>
env.eventBus.publish(LoginEvent(user, request, implicitly[Messages]))
env.authenticatorService.init(authenticator).flatMap(cookie =>
env.authenticatorService.embed(cookie.copy(secure = request.secure), Redirect(routes.Application.index(Map())))
)
}
case false => Future.successful(Redirect(routes.AuthController.signIn()).flashing("error" -> Messages("invalid.credentials")))
}
}
)
}
def signOut = SecuredAction.async { implicit request =>
val result = Redirect(routes.Application.index(Map()))
env.eventBus.publish(LogoutEvent(request.identity, request, request2Messages))
env.authenticatorService.discard(request.authenticator, result)
}
}

View File

@@ -0,0 +1,31 @@
package controllers
import com.mohiva.play.silhouette.api.Silhouette
import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator
import models.User
import play.api.mvc.{Result, RequestHeader, Results}
import views.html.DefaultRequest
import scala.concurrent.Future
import scala.language.implicitConversions
trait AuthenticatedControllerLowPriorityImplicits[T, C]{
self: AuthenticatedController =>
protected object secureRequestConversion{
implicit def securedRequestToUserAwareRequest(implicit req: SecuredRequest[_]): DefaultRequest = UserAwareRequest(Some(req.identity), authenticator = Some(req.authenticator), req.request)
}
}
abstract class AuthenticatedController extends Silhouette[User, CookieAuthenticator] with AuthenticatedControllerLowPriorityImplicits[User, CookieAuthenticator]{
override protected def onNotAuthenticated(request: RequestHeader): Option[Future[Result]] = Some(Future.successful(Redirect(routes.AuthController.signIn())))
object ReadAction extends SecuredActionBuilder with Results {
}
def AdminAction: SecuredActionBuilder = ???
}

View File

@@ -0,0 +1,143 @@
package controllers
import java.net.URLEncoder
import com.google.inject.Inject
import com.ysoft.odc._
import controllers.DependencyCheckReportsParser.Result
import models.PlainLibraryIdentifier
import play.api.Logger
import play.api.cache.CacheApi
import play.twirl.api.Html
import scala.util.{Failure, Success, Try}
sealed trait Filter{
def selector: Option[String]
def subReports(r: Result): Option[Result]
def filters: Boolean
def descriptionHtml: Html
def descriptionText: String
}
private final case class ProjectFilter(project: ReportInfo) extends Filter{
override def filters: Boolean = true
override def descriptionHtml: Html = views.html.filters.project(project)
override def descriptionText: String = s"project ${friendlyProjectName(project)}"
override def subReports(r: Result): Option[Result] = {
@inline def reportInfo = project
def f[T](m: Map[ReportInfo, T]): Map[String, T] = (
if(reportInfo.subprojectNameOption.isEmpty) m.filter(_._1.projectId == project.projectId) else m.get(reportInfo).fold(Map.empty[ReportInfo, T])(x => Map(reportInfo -> x))
).map{case (k, v) => k.fullId -> v}
val newFlatReports = f(r.flatReports)
val newFailedAnalysises = f(r.failedAnalysises)
if(newFlatReports.isEmpty && newFailedAnalysises.isEmpty) None
else Some(Result(bareFlatReports = newFlatReports, bareFailedAnalysises = newFailedAnalysises, projects = r.projects))
}
override def selector = Some(s"project:${project.fullId}")
}
private final case class TeamFilter(team: Team) extends Filter{
override def filters: Boolean = true
override def subReports(r: Result): Option[Result] = {
val reportInfoByFriendlyProjectName = r.projectsReportInfo.ungroupedReportsInfo.map(ri => friendlyProjectName(ri) -> ri).toSeq.groupBy(_._1).mapValues{
case Seq((_, ri)) => ri
case other => sys.error("some duplicate value: "+other)
}.map(identity)
val reportInfos = team.projectNames.map(reportInfoByFriendlyProjectName)
def submap[T](m: Map[String, T]) = reportInfos.toSeq.flatMap(ri => m.get(ri.fullId).map(ri.fullId -> _) ).toMap
Some(Result(
bareFlatReports = submap(r.bareFlatReports),
bareFailedAnalysises = submap(r.bareFailedAnalysises),
projects = r.projects
))
}
override def descriptionHtml: Html = views.html.filters.team(team.id)
override def descriptionText: String = s"team ${team.name}"
override def selector = Some(s"team:${team.id}")
}
object NoFilter extends Filter{
override def filters: Boolean = false
override val descriptionHtml: Html = views.html.filters.all()
override def descriptionText: String = "all projects"
override def subReports(r: Result): Option[Result] = Some(r)
override def selector: Option[String] = None
}
private final case class BadFilter(pattern: String) extends Filter{
override def filters: Boolean = true
override def subReports(r: Result): Option[Result] = None
override def descriptionHtml: Html = Html("<b>bad filter</b>")
override def descriptionText: String = "bad filter"
override def selector: Option[String] = Some(pattern)
}
object DependencyCheckReportsParser{
final case class ResultWithSelection(result: Result, projectsWithSelection: ProjectsWithSelection)
final case class Result(bareFlatReports: Map[String, Analysis], bareFailedAnalysises: Map[String, Throwable], projects: Projects){
lazy val projectsReportInfo = new ProjectsWithReports(projects, bareFlatReports.keySet ++ bareFailedAnalysises.keySet)
lazy val flatReports: Map[ReportInfo, Analysis] = bareFlatReports.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v}
lazy val failedAnalysises: Map[ReportInfo, Throwable] = bareFailedAnalysises.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v}
lazy val allDependencies = flatReports.toSeq.flatMap(r => r._2.dependencies.map(_ -> r._1))
lazy val groupedDependencies = allDependencies.groupBy(_._1.hashes).values.map(GroupedDependency(_)).toSeq
lazy val groupedDependenciesByPlainLibraryIdentifier: Map[PlainLibraryIdentifier, Set[GroupedDependency]] =
groupedDependencies.toSet.flatMap((grDep: GroupedDependency) => grDep.plainLibraryIdentifiers.map(_ -> grDep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
lazy val vulnerableDependencies = groupedDependencies.filter(_.vulnerabilities.nonEmpty)
private val ProjectSelectorPattern = """^project:(.*)$""".r
private val TeamSelectorPattern = """^team:(.*)$""".r
private def parseFilter(filter: String): Filter = filter match {
case ProjectSelectorPattern(project) => ProjectFilter(projectsReportInfo.reportIdToReportInfo(project))
case TeamSelectorPattern(team) => TeamFilter(projects.teamById(team))
case other => BadFilter(other)
}
def selection(selectorOption: Option[String]): Option[ResultWithSelection] = {
val filter = selectorOption.map(parseFilter).getOrElse(NoFilter)
filter.subReports(this).map{ result =>
ResultWithSelection(
result = result,
projectsWithSelection = ProjectsWithSelection(filter = filter, projectsWithReports = projectsReportInfo, teams = projects.teamSet)
)
}
}
}
}
final class DependencyCheckReportsParser @Inject() (cache: CacheApi, projects: Projects) {
def parseReports(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)]) = {
val rid = math.random.toString // for logging
@volatile var parseFailedForSomeAnalysis = false
val deepReportsTriesIterable: Iterable[Map[String, Try[Analysis]]] = for((k, (build, data, log)) <- successfulResults) yield {
Logger.debug(data.flatFilesWithPrefix(s"$k/").keySet.toSeq.sorted.toString)
val flat = data.flatFilesWithPrefix(s"$k/")
(for((k, v) <- flat.par) yield {
val analysisKey = URLEncoder.encode(s"analysis/parsedXml/${build.buildResultKey}/${k}", "utf-8")
Logger.debug(s"[$rid] analysisKey: $analysisKey")
val analysisTry = cache.getOrElse(analysisKey)(Try{OdcParser.parseXmlReport(v)})
analysisTry match{
case Success(e) => // nothing
case Failure(e) =>
if(!parseFailedForSomeAnalysis){
Logger.error(s"[$rid] Cannot parse $k: ${new String(v, "utf-8")}", e)
parseFailedForSomeAnalysis = true
}
}
k -> analysisTry
}).seq
}
val deepReportsAndFailuresIterable = deepReportsTriesIterable.map { reports =>
val (successfulReportsTries, failedReportsTries) = reports.partition(_._2.isSuccess)
val successfulReports = successfulReportsTries.mapValues(_.asInstanceOf[Success[Analysis]].value).map(identity)
val failedReports = failedReportsTries.mapValues(_.asInstanceOf[Failure[Analysis]].exception).map(identity)
(successfulReports, failedReports)
}
val deepSuccessfulReports = deepReportsAndFailuresIterable.map(_._1).toSeq
val failedAnalysises = deepReportsAndFailuresIterable.map(_._2).toSeq.flatten.toMap
val flatReports = deepSuccessfulReports.flatten.toMap
Logger.debug(s"[$rid] parse finished")
Result(flatReports, failedAnalysises, projects)
}
}

View File

@@ -0,0 +1,106 @@
package controllers
import com.github.nscala_time.time.Imports._
import com.google.inject.Inject
import com.google.inject.name.Named
import com.ysoft.odc.Checks._
import com.ysoft.odc._
import org.joda.time.DateTimeConstants
import play.api.Logger
import play.api.i18n.{I18nSupport, MessagesApi}
import play.api.mvc.RequestHeader
import play.twirl.api.Html
import views.html.DefaultRequest
import scala.concurrent.{ExecutionContext, Future}
final case class MissingGavExclusions(exclusionsSet: Set[Exclusion]){
def isExcluded(groupedDependency: GroupedDependency) = exclusionsSet.exists(_.matches(groupedDependency))
}
final class DependencyCheckReportsProcessor @Inject() (
@Named("bamboo-server-url") val server: String,
dependencyCheckReportsParser: DependencyCheckReportsParser,
@Named("missing-GAV-exclusions") missingGAVExclusions: MissingGavExclusions,
val messagesApi: MessagesApi
) extends I18nSupport {
private def parseDateTime(dt: String): DateTime = {
if(dt.forall(_.isDigit)){
new DateTime(dt.toLong) // TODO: timezone (I don't care much, though)
}else{
val formatter = DateTimeFormat.forPattern("dd/MM/yyyy HH:mm:ss") // TODO: timezone (I don't care much, though)
formatter.parseDateTime(dt)
}
}
@deprecated("use HTML output instead", "SNAPSHOT") private val showDependencies: (Seq[GroupedDependency]) => Seq[String] = {
_.map { s =>
s.dependencies.map { case (dep, projects) => s"${dep.fileName} @ ${projects.toSeq.sorted.map(friendlyProjectName).mkString(", ")}" }.mkString(", ") + " " + s.hashes
}
}
def processResults(
resultsFuture: Future[(Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable])],
requiredVersions: Map[String, Int]
)(implicit requestHeader: DefaultRequest, snoozesInfo: SnoozesInfo, executionContext: ExecutionContext) = try{
for((successfulResults, failedResults) <- resultsFuture) yield{
val reportResult = dependencyCheckReportsParser.parseReports(successfulResults)
import reportResult.{allDependencies, failedAnalysises, flatReports, groupedDependencies, vulnerableDependencies}
val now = DateTime.now
val oldReportThreshold = now - 1.day
val cveTimestampThreshold = now - (if(now.dayOfWeek().get == DateTimeConstants.MONDAY) 4.days else 2.days )
val ScanChecks: Seq[Map[ReportInfo, Analysis] => Option[Warning]] = Seq(
differentValues("scan infos", "scan-info", WarningSeverity.Warning)(_.groupBy(_._2.scanInfo).mapValues(_.keys.toIndexedSeq.sorted)),
badValues("old-reports", "old reports", WarningSeverity.Warning)((_, a) => if(a.reportDate < oldReportThreshold) Some(Html(a.reportDate.toString)) else None),
badValues("bad-cve-data", "old or no CVE data", WarningSeverity.Warning){(_, analysis) =>
(analysis.scanInfo.xml \\ "timestamp").map(_.text).filterNot(_ == "").map(parseDateTime) match {
case Seq() => Some(Html("no data"))
case timestamps =>
val newestTimestamp = timestamps.max
val oldestTimestamp = timestamps.min
if(newestTimestamp < cveTimestampThreshold) Some(Html(newestTimestamp.toString))
else None
}
}
)
val GroupedDependenciesChecks = Seq[Seq[GroupedDependency] => Option[Warning]](
badGroupedDependencies("unidentified-dependencies", "unidentified dependencies", WarningSeverity.Info)(_.filter(_.dependencies.exists(_._1.identifiers.isEmpty)))(show = showDependencies, exclusions = missingGAVExclusions.exclusionsSet),
badGroupedDependencies("different-identifier-sets", "different identifier sets", WarningSeverity.Info)(_.filter(_.dependencies.groupBy(_._1.identifiers).size > 1).toIndexedSeq)(),
badGroupedDependencies("different-evidence", "different evidence", WarningSeverity.Info)(_.filter(_.dependencies.groupBy(_._1.evidenceCollected).size > 1).toIndexedSeq)(show = x => Some(views.html.warnings.groupedDependencies(x))),
badGroupedDependencies("missing-gav", "missing GAV", WarningSeverity.Info)(_.filter(_.identifiers.filter(_.identifierType == "maven").isEmpty))(show = showDependencies, exclusions = missingGAVExclusions.exclusionsSet)
)
val unknownIdentifierTypes = allDependencies.flatMap(_._1.identifiers.map(_.identifierType)).toSet -- Set("maven", "cpe")
val failedReports = successfulResults.filter(x => x._2._1.state != "Successful" || x._2._1.buildState != "Successful")
val extraWarnings = Seq[Option[Warning]](
if(failedReports.size > 0) Some(IdentifiedWarning("failed-reports", views.html.warnings.failedReports(failedReports.values.map{case (b, _ ,_) => b}.toSet, server), WarningSeverity.Error)) else None,
if(unknownIdentifierTypes.size > 0) Some(IdentifiedWarning("unknown-identifier-types", views.html.warnings.unknownIdentifierType(unknownIdentifierTypes), WarningSeverity.Info)) else None,
{
val emptyResults = successfulResults.filter{case (k, (_, dir, _)) => dir.flatFiles.size < 1}
if(emptyResults.nonEmpty) Some(IdentifiedWarning("empty-results", views.html.warnings.emptyResults(emptyResults.values.map{case (build, _, _) => build}.toSeq, server), WarningSeverity.Warning)) else None
},
{
val resultsWithErrorMessages = successfulResults.filter{case (k, (_, _, log)) => log.dataString.lines.exists(l => (l.toLowerCase startsWith "error") || (l.toLowerCase contains "[error]"))}
if(resultsWithErrorMessages.nonEmpty) Some(IdentifiedWarning("results-with-error-messages", views.html.warnings.resultsWithErrorMessages(resultsWithErrorMessages.values.map{case (build, _, _) => build}.toSeq, server), WarningSeverity.Error)) else None
},
if(failedResults.isEmpty) None else Some(IdentifiedWarning("failed-results", views.html.warnings.failedResults(failedResults), WarningSeverity.Error)),
if(requiredVersions.isEmpty) None else Some(IdentifiedWarning("required-versions", views.html.warnings.textWarning("You have manually requested results for some older version."), WarningSeverity.Warning)),
if(failedAnalysises.isEmpty) None else Some(IdentifiedWarning("failed-analysises", views.html.warnings.textWarning(s"Some reports failed to parse: ${failedAnalysises.keySet}"), WarningSeverity.Error))
).flatten
val scanWarnings = ScanChecks.flatMap(_(flatReports))
val groupedDependenciesWarnings = GroupedDependenciesChecks.flatMap(_(groupedDependencies))
val allWarnings = scanWarnings ++ groupedDependenciesWarnings ++ extraWarnings
// TODO: log analysis
// TODO: related dependencies
(vulnerableDependencies, allWarnings, groupedDependencies)
}
}finally{
Logger.debug("Reports processed")
}
}

View File

@@ -0,0 +1,40 @@
package controllers
import com.github.nscala_time.time.Imports._
import com.google.inject.Inject
import com.ysoft.odc.Downloader
import play.api.cache.CacheApi
import scala.concurrent.{ExecutionContext, Future}
import scala.reflect.ClassTag
import scala.util.Success
class ProjectReportsProvider @Inject() (
bambooDownloader: Downloader,
cache: CacheApi,
projects: Projects
)(implicit executionContext: ExecutionContext){
private def bambooCacheKey(versions: Map[String, Int]) = "bamboo/results/" + versions.toSeq.sorted.map{case (k, v) => k.getBytes("utf-8").mkString("-") + ":" + v}.mkString("|")
def purgeCache(versions: Map[String, Int]) = cache.remove(bambooCacheKey(versions))
private def getOrElseFuture[T: ClassTag]
(name: String, expiration: scala.concurrent.duration.Duration = scala.concurrent.duration.Duration.Inf)
(f: => Future[T])
(implicit executionContext: ExecutionContext): Future[T] =
{
cache.get[T](name).map(Future.successful).getOrElse(
f.andThen{
case Success(value) =>cache.set(name, value, expiration)
}
)
}
def resultsForVersions(versions: Map[String, Int]) = {
def get = {val time = DateTime.now; bambooDownloader.downloadProjectReports(projects.projectSet, versions).map(time -> _)}
val allFuture = getOrElseFuture(bambooCacheKey(versions)){println("CACHE MISS"); get}
(allFuture.map(_._1), allFuture.map(_._2))
}
}

View File

@@ -0,0 +1,52 @@
package controllers
import javax.inject.Inject
import play.api.Configuration
class Projects @Inject() (configuration: Configuration) {
import scala.collection.JavaConversions._
val projectMap = {
val projectsConfig = configuration.getObject("yssdc.projects").getOrElse(sys.error("yssdc.projects is not set")).toConfig
projectsConfig.entrySet().map( k => k.getKey -> projectsConfig.getString(k.getKey)).toMap
}
val projectSet = projectMap.keySet
val teamIdSet = configuration.getStringSeq("yssdc.teams").getOrElse(sys.error("yssdc.teams is not set")).map(TeamId).toSet
private val teamsByIds = teamIdSet.map(t => t.id -> t).toMap
val teamLeaders = {
import scala.collection.JavaConversions._
configuration.getObject("yssdc.teamLeaders").getOrElse(sys.error("yssdc.teamLeaders is not set")).map{case(k, v) =>
TeamId(k) -> v.unwrapped().asInstanceOf[String]
}
}
{
val extraTeams = teamLeaders.keySet -- teamIdSet
if(extraTeams.nonEmpty){
sys.error(s"Some unexpected teams: $extraTeams")
}
}
def existingTeamId(s: String): TeamId = teamsByIds(s)
val projectToTeams = configuration.getObject("yssdc.projectsToTeams").get.mapValues{_.unwrapped().asInstanceOf[java.util.List[String]].map(c =>
existingTeamId(c)
).toSet}.map(identity)
val projectAndTeams = projectToTeams.toSeq.flatMap{case (project, teams) => teams.map(team => (project, team))}
val teamsToProjects = projectAndTeams.groupBy(_._2).mapValues(_.map(_._1).toSet).map(identity)
val teamsById: Map[String, Team] = for{
(team, projectNames) <- teamsToProjects
} yield team.id -> Team(
id = team.id,
name = team.name,
leader = teamLeaders(team),
projectNames = projectNames
)
def teamById(id: String) = teamsById(id)
def teamSet = teamsById.values.toSet
}

View File

@@ -0,0 +1,58 @@
package controllers
final case class ReportInfo(
projectId: String,
projectName: String,
fullId: String,
subprojectNameOption: Option[String]
) extends Ordered[ReportInfo] {
import scala.math.Ordered.orderingToOrdered
override def compare(that: ReportInfo): Int = ((projectName, subprojectNameOption, fullId)) compare ((that.projectName, that.subprojectNameOption, that.fullId))
// It seems to be a good idea to have a custom equals and hashCode for performance reasons
override def equals(other: Any): Boolean = other match {
case other: ReportInfo => fullId == other.fullId
case _ => false
}
override def hashCode(): Int = 517+fullId.hashCode
}
object ProjectsWithReports{
private val RestMessBeginRegexp = """^/Report results-XML/""".r
private val RestMessEndRegexp = """/(target/)?dependency-check-report\.xml$""".r
}
class ProjectsWithReports (val projects: Projects, val reports: Set[String]) {
import ProjectsWithReports._
val reportIdToReportInfo = {
val reportsMap = reports.map{ unfriendlyName =>
val (baseName, theRest) = unfriendlyName.span(_ != '/')
val removeLeadingMess = RestMessBeginRegexp.replaceAllIn(_: String, "")
val removeTrailingMess = RestMessEndRegexp.replaceAllIn(_: String, "")
val removeMess = removeLeadingMess andThen removeTrailingMess
val subProjectOption = Some(removeMess(theRest)).filter(_ != "")
subProjectOption.fold(baseName)(baseName+"/"+_)
unfriendlyName -> ReportInfo(
projectId = baseName,
fullId = unfriendlyName,
projectName = projects.projectMap(baseName),
subprojectNameOption = subProjectOption
)
}.toMap
reportsMap ++ reportsMap.values.map(r => r.projectId -> ReportInfo(projectId = r.projectId, fullId = r.projectId, subprojectNameOption = None, projectName = r.projectName))
}
val ungroupedReportsInfo = reportIdToReportInfo.values.toSet
}

View File

@@ -0,0 +1,14 @@
package controllers
final case class TeamId(id: String) extends AnyVal {
def name = id
}
final case class Team(id: String, name: String, leader: String, projectNames: Set[String])
// TODO: rename to something more sane. It is maybe rather FilteringData now.
final case class ProjectsWithSelection(filter: Filter, projectsWithReports: ProjectsWithReports, teams: Set[Team]) {
def isProjectSpecified: Boolean = filter.filters
def selectorString = filter.selector
def projectNameText: String = filter.descriptionText
}

View File

@@ -0,0 +1,226 @@
package controllers
import com.github.nscala_time.time.Imports._
import com.google.inject.Inject
import com.google.inject.name.Named
import com.ysoft.odc.{ArtifactFile, ArtifactItem}
import models.{Library, LibraryTag}
import org.joda.time.DateTime
import play.api.i18n.MessagesApi
import play.twirl.api.Txt
import services.{LibrariesService, LibraryTagAssignmentsService, OdcService, TagsService}
import views.html.DefaultRequest
import scala.concurrent.{ExecutionContext, Future}
object Statistics{
case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency]){
def vulnerableRatio = vulnerableDependencies.size.toDouble / dependencies.size.toDouble
lazy val vulnerabilities: Set[Vulnerability] = dependencies.flatMap(_.vulnerabilities)
lazy val vulnerabilitiesToDependencies: Map[Vulnerability, Set[GroupedDependency]] = vulnerableDependencies.flatMap(dep =>
dep.vulnerabilities.map(vuln => (vuln, dep))
).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
lazy val vulnerableDependencies = dependencies.filter(_.isVulnerable)
lazy val (dependenciesWithCpe, dependenciesWithoutCpe) = dependencies.partition(_.hasCpe)
lazy val cpeRatio = dependenciesWithCpe.size.toDouble / dependencies.size.toDouble
lazy val weaknesses = vulnerabilities.flatMap(_.cweOption)
lazy val weaknessesFrequency = computeWeaknessesFrequency(vulnerabilities)
}
case class TagStatistics(tagRecord: (Int, LibraryTag), stats: LibDepStatistics){
def tag: LibraryTag = tagRecord._2
def tagId: Int = tagRecord._1
}
def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0)
}
import controllers.Statistics._
class Statistics @Inject() (
reportsParser: DependencyCheckReportsParser,
reportsProcessor: DependencyCheckReportsProcessor,
projectReportsProvider: ProjectReportsProvider,
dependencyCheckReportsParser: DependencyCheckReportsParser,
librariesService: LibrariesService,
tagsService: TagsService,
odcService: OdcService,
libraryTagAssignmentsService: LibraryTagAssignmentsService,
@Named("missing-GAV-exclusions") missingGAVExclusions: MissingGavExclusions,
projects: Projects,
val env: AuthEnv
)(implicit val messagesApi: MessagesApi, executionContext: ExecutionContext) extends AuthenticatedController {
private val versions = Map[String, Int]()
private def notFound()(implicit req: DefaultRequest) = {
NotFound(views.html.defaultpages.notFound("GET", req.uri))
}
import secureRequestConversion._
private def select(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], selectorOption: Option[String]) = dependencyCheckReportsParser.parseReports(successfulResults).selection(selectorOption)
def searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String]) = ReadAction.async{ implicit req =>
if(versionlessCpes.isEmpty){
Future.successful(notFound())
}else{
val now = DateTime.now()
val oldDataThreshold = 2.days
val lastDbUpdateFuture = odcService.loadLastDbUpdate()
val isOldFuture = lastDbUpdateFuture.map{ lastUpdate => now - oldDataThreshold > lastUpdate}
versionOption match {
case Some(version) =>
for {
res1 <- Future.traverse(versionlessCpes) { versionlessCpe => odcService.findRelevantCpes(versionlessCpe, version) }
vulnIds = res1.flatten.map(_.vulnerabilityId).toSet
vulns <- Future.traverse(vulnIds)(id => odcService.getVulnerabilityDetails(id).map(_.get))
isOld <- isOldFuture
} yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
vulnsAndVersionOption = Some((vulns, version)),
cpes = versionlessCpes,
isDbOld = isOld
))
case None =>
for(isOld <- isOldFuture) yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
vulnsAndVersionOption = None,
cpes = versionlessCpes,
isDbOld = isOld
))
}
}
}
def basic(projectOption: Option[String]) = ReadAction.async{ implicit req =>
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
resultsFuture flatMap { case (successfulResults, failedResults) =>
select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection =>
val tagsFuture = tagsService.all
val parsedReports = selection.result
for{
tagStatistics <- statisticsForTags(parsedReports, tagsFuture)
} yield Ok(views.html.statistics.basic(
tagStatistics = tagStatistics,
projectsWithSelection = selection.projectsWithSelection,
parsedReports = parsedReports
))
}
}
}
def statisticsForTags(parsedReports: DependencyCheckReportsParser.Result, tagsFuture: Future[Seq[(Int, LibraryTag)]]): Future[Seq[Statistics.TagStatistics]] = {
val librariesFuture = librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet)
val libraryTagAssignmentsFuture = librariesFuture.flatMap{libraries => libraryTagAssignmentsService.forLibraries(libraries.values.map(_._1).toSet)}
val tagsToLibrariesFuture = libraryTagAssignmentsService.tagsToLibraries(libraryTagAssignmentsFuture)
val librariesToDependencies = parsedReports.groupedDependenciesByPlainLibraryIdentifier
for{
librariesById <- librariesFuture.map(_.values.toMap)
tagsToLibraries <- tagsToLibrariesFuture
tags <- tagsFuture
} yield tags.flatMap{case tagRecord @ (tagId, tag) =>
val libraryAssignments = tagsToLibraries(tagId)
val tagLibraries = libraryAssignments.map(a => a.libraryId -> librariesById(a.libraryId))
val tagDependencies: Set[GroupedDependency] = tagLibraries.flatMap{case (_, lib) => librariesToDependencies(lib.plainLibraryIdentifier)}
// TODO: vulnerabilities in the past
if(tagLibraries.isEmpty) None
else Some(TagStatistics(tagRecord = tagRecord, stats = LibDepStatistics(libraries = tagLibraries, dependencies = tagDependencies)))
}
}
def vulnerabilities(projectOption: Option[String], tagIdOption: Option[Int]) = ReadAction.async {implicit req =>
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
resultsFuture flatMap { case (successfulResults, failedResults) =>
select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection =>
val parsedReports = selection.result
for{
libraries <- librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet)
tagOption <- tagIdOption.fold[Future[Option[(Int, LibraryTag)]]](Future.successful(None))(tagId => tagsService.getById(tagId).map(Some(_)))
statistics <- tagOption.fold(Future.successful(LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.values.toSet))){ tag =>
statisticsForTags(parsedReports, Future.successful(Seq(tag))).map{
case Seq(TagStatistics(_, stats)) => stats // statisticsForTags is designed for multiple tags, but we have just one…
case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set()) // We don't want to crash when no dependencies are there…
}
}
} yield Ok(views.html.statistics.vulnerabilities(
projectsWithSelection = selection.projectsWithSelection,
tagOption = tagOption,
statistics = statistics
))
}
}
}
def vulnerability(name: String, projectOption: Option[String]) = ReadAction.async { implicit req =>
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
resultsFuture flatMap { case (successfulResults, failedResults) =>
select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection =>
val relevantReports = selection.result
val vulns = relevantReports.vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(vuln => (vuln, dep))).groupBy(_._1.name).mapValues{case vulnsWithDeps =>
val (vulnSeq, depSeq) = vulnsWithDeps.unzip
val Seq(vuln) = vulnSeq.toSet.toSeq // Will fail when there are more different descriptions for one vulnerability…
vuln -> depSeq.toSet
}// .map(identity) // The .map(identity) materializes lazily mapped Map (because .mapValues is lazy). I am, however, unsure if this is a good idea. Probably not.
vulns.get(name).fold(Future.successful(Ok(views.html.statistics.vulnerabilityNotFound(
name = name,
projectsWithSelection = selection.projectsWithSelection
)))){ case (vuln, vulnerableDependencies) =>
for(
plainLibs <- librariesService.byPlainLibraryIdentifiers(vulnerableDependencies.flatMap(_.plainLibraryIdentifiers)).map(_.keySet)
) yield Ok(views.html.statistics.vulnerability(
vulnerability = vuln,
affectedProjects = vulnerableDependencies.flatMap(dep => dep.projects.map(proj => (proj, dep))).groupBy(_._1).mapValues(_.map(_._2)),
vulnerableDependencies = vulnerableDependencies,
affectedLibraries = plainLibs,
projectsWithSelection = selection.projectsWithSelection
))
}
}
}
}
def vulnerableLibraries(project: Option[String]) = ReadAction.async { implicit req =>
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
resultsFuture flatMap { case (successfulResults, failedResults) =>
select(successfulResults, project).fold(Future.successful(notFound())){selection =>
val reports = selection.result
Future.successful(Ok(views.html.statistics.vulnerableLibraries(
projectsWithSelection = selection.projectsWithSelection,
vulnerableDependencies = reports.vulnerableDependencies,
allDependenciesCount = reports.groupedDependencies.size
)))
}
}
}
def allLibraries(project: Option[String]) = ReadAction.async { implicit req =>
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
resultsFuture flatMap { case (successfulResults, failedResults) =>
select(successfulResults, project).fold(Future.successful(notFound())){selection =>
Future.successful(Ok(views.html.statistics.allLibraries(
projectsWithSelection = selection.projectsWithSelection,
allDependencies = selection.result.groupedDependencies
)))
}
}
}
def allGavs(project: Option[String]) = ReadAction.async { implicit req =>
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
resultsFuture flatMap { case (successfulResults, failedResults) =>
select(successfulResults, project).fold(Future.successful(notFound())){selection =>
Future.successful(Ok(Txt(
selection.result.groupedDependencies.flatMap(_.mavenIdentifiers).toSet.toIndexedSeq.sortBy((id: Identifier) => (id.identifierType, id.name)).map(id => id.name.split(':') match {
case Array(g, a, v) =>
s""""${id.identifierType}", "$g", "$a", "$v", "${id.url}" """
}).mkString("\n")
)))
}
}
}
}

View File

@@ -0,0 +1,37 @@
import com.mohiva.play.silhouette.api.Environment
import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator
import models.{User, SnoozeInfo}
/**
* Created by user on 7/15/15.
*/
package object controllers {
// Imports for all templates. Those could be added directly to the template files, but IntelliJ IDEA does not like it.
type Dependency = com.ysoft.odc.Dependency
type Build = com.ysoft.odc.Build
type GroupedDependency = com.ysoft.odc.GroupedDependency
type Vulnerability = com.ysoft.odc.Vulnerability
type Identifier = com.ysoft.odc.Identifier
type DateTime = org.joda.time.DateTime
type SnoozesInfo = Map[String, SnoozeInfo]
type AuthEnv = Environment[User, CookieAuthenticator]
val NormalUrlPattern = """^(http(s?)|ftp(s?))://.*""".r
val TooGenericDomains = Set("sourceforge.net", "github.com", "github.io")
/* def friendlyProjectName(unfriendlyName: String) = {
val (baseName, theRest) = unfriendlyName.span(_ != '/')
//theRest.drop(1)
val removeLeadingMess = RestMessBeginRegexp.replaceAllIn(_: String, "")
val removeTrailingMess = RestMessEndRegexp.replaceAllIn(_: String, "")
val removeMess = removeLeadingMess andThen removeTrailingMess
val subProjectOption = Some(removeMess(theRest)).filter(_ != "")
subProjectOption.fold(baseName)(baseName+"/"+_)
}*/
def friendlyProjectName(reportInfo: ReportInfo) = reportInfo.subprojectNameOption.fold(reportInfo.projectName)(reportInfo.projectName+": "+_)
}

View File

@@ -0,0 +1,21 @@
package controllers
import controllers.WarningSeverity.WarningSeverity
import play.twirl.api.Html
object WarningSeverity extends Enumeration {
type WarningSeverity = Value
// Order is important
val Info = Value("info")
val Warning = Value("warning")
val Error = Value("error")
}
sealed abstract class Warning {
def html: Html
def id: String
def allowSnoozes = true
def severity: WarningSeverity
}
final case class IdentifiedWarning(id: String, html: Html, severity: WarningSeverity) extends Warning

View File

@@ -0,0 +1,33 @@
package models
import scala.concurrent.duration._
import com.mohiva.play.silhouette.api.LoginInfo
import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator
import models.profile.MappedJdbcType
import models.jodaSupport._
import models.profile.api._
import org.joda.time.DateTime
import slick.lifted.{ProvenShape, Tag}
import scala.concurrent.duration.FiniteDuration
class CookieAuthenticators(tag: Tag) extends Table[CookieAuthenticator](tag, "cookie_authenticators") {
private implicit val FiniteDurationType = MappedJdbcType.base[FiniteDuration, Long](_.toSeconds, FiniteDuration.apply(_, SECONDS))
def id = column[String]("id")
def providerId = column[String]("provider_id")
def providerKey = column[String]("provider_key")
def lastUsedDateTime = column[DateTime]("last_used")
def expirationDateTime = column[DateTime]("expiration")
def idleTimeout = column[FiniteDuration]("idle_timeout").?
def cookieMaxAge = column[FiniteDuration]("cookie_max_age").?
def fingerprint = column[String]("fingerprint").?
def loginInfo = (providerId, providerKey) <> (LoginInfo.tupled, LoginInfo.unapply)
override def * : ProvenShape[CookieAuthenticator] = (id, loginInfo, lastUsedDateTime, expirationDateTime, idleTimeout, cookieMaxAge, fingerprint) <> ((CookieAuthenticator.apply _).tupled, CookieAuthenticator.unapply)
}

52
app/models/Library.scala Normal file
View File

@@ -0,0 +1,52 @@
package models
import models.profile.MappedJdbcType
import models.profile.api._
import slick.lifted.Tag
abstract sealed class LibraryType(val name: String){
override final def toString: String = name
}
object LibraryType{
case object Maven extends LibraryType("maven")
case object DotNet extends LibraryType("dotnet")
val All = Set(Maven, DotNet)
val ByName = All.map(x => x.name -> x).toMap
implicit val libraryTypeMapper = MappedJdbcType.base[LibraryType, String](_.name, LibraryType.ByName)
}
final case class Library(plainLibraryIdentifier: PlainLibraryIdentifier, classified: Boolean)
final case class PlainLibraryIdentifier(libraryType: LibraryType, libraryIdentifier: String){
override def toString: String = s"$libraryType:$libraryIdentifier"
}
object PlainLibraryIdentifier extends ((LibraryType, String) => PlainLibraryIdentifier) {
def fromString(id: String) = {
val (libraryType, libraryNameWithColon) = id.span(_ != ':')
if(libraryNameWithColon(0) != ':'){
sys.error("Expected colon")
}
val libraryName = libraryNameWithColon.drop(1)
PlainLibraryIdentifier(
libraryType = LibraryType.ByName(libraryType),
libraryIdentifier = libraryName
)
}
}
class Libraries(tag: Tag) extends Table[(Int, Library)](tag, "library") {
import LibraryType.libraryTypeMapper
def id = column[Int]("id", O.PrimaryKey)
def libraryType = column[LibraryType]("library_type")
def libraryIdentifier = column[String]("identifier")
def classified = column[Boolean]("classified")
def plainLibraryIdentifierUnmapped = (libraryType, libraryIdentifier)
def plainLibraryIdentifier = plainLibraryIdentifierUnmapped <> (PlainLibraryIdentifier.tupled, PlainLibraryIdentifier.unapply)
def base = (plainLibraryIdentifier, classified) <> (Library.tupled, Library.unapply)
def * = (id, base)
}

View File

@@ -0,0 +1,16 @@
package models
import models.profile.api._
import slick.lifted.Tag
final case class LibraryTag (name: String, note: Option[String], warningOrder: Option[Int])
class LibraryTags(tag: Tag) extends Table[(Int, LibraryTag)](tag, "library_tag") {
def id = column[Int]("id", O.PrimaryKey)
def name = column[String]("name")
def note = column[Option[String]]("note")
def warningOrder = column[Option[Int]]("warning_order")
def base = (name, note, warningOrder) <> (LibraryTag.tupled, LibraryTag.unapply)
def * = (id, base)
}

View File

@@ -0,0 +1,19 @@
package models
import models.profile.api._
import slick.lifted.Tag
final case class LibraryTagPair(libraryId: Int, tagId: Int)
final case class LibraryTagAssignment(libraryTagPair: LibraryTagPair, contextDependent: Boolean){
def libraryId = libraryTagPair.libraryId
def tagId = libraryTagPair.tagId
}
class LibraryTagAssignments(tag: Tag) extends Table[LibraryTagAssignment](tag, "library_to_library_tag") {
def libraryId = column[Int]("library_id")
def libraryTagId = column[Int]("library_tag_id")
def contextDependent = column[Boolean]("context_dependent")
def libraryTagPair = (libraryId, libraryTagId) <> (LibraryTagPair.tupled, LibraryTagPair.unapply)
def * = (libraryTagPair, contextDependent) <> (LibraryTagAssignment.tupled, LibraryTagAssignment.unapply)
}

44
app/models/Snooze.scala Normal file
View File

@@ -0,0 +1,44 @@
package models
import models.jodaSupport._
import models.profile.api._
import org.joda.time.LocalDate
import play.api.data.Form
import slick.lifted.Tag
case class Snooze(until: LocalDate, snoozedObjectId: String, reason: String)
case class ObjectSnooze(until: LocalDate, reason: String){
def toSnooze(objectId: String) = Snooze(until, objectId, reason)
}
class Snoozes(tag: Tag) extends Table[(Int, Snooze)](tag, "snooze") {
def id = column[Int]("id", O.PrimaryKey)
def until = column[LocalDate]("until")
def snoozedObjectId = column[String]("snoozed_object_identifier")
def reason = column[String]("reason")
def base = (until, snoozedObjectId, reason) <> (Snooze.tupled, Snooze.unapply)
def * = (id, base)
}
case class SnoozeInfo(form: Form[ObjectSnooze], snoozes: Seq[(Int, Snooze)]){
def shouldCollapse(default: Boolean) = {
shouldExpandForm match {
case true => false
case false =>
isSnoozed match {
case true => true
case false => default
}
}
}
def isSnoozed = snoozes.nonEmpty
def shouldExpandForm = form.hasErrors || form.hasGlobalErrors
def adjustForm(f: Form[ObjectSnooze] => Form[ObjectSnooze]): SnoozeInfo = copy(form = f(form))
def adjustSnoozes(f: Seq[(Int, Snooze)] => Seq[(Int, Snooze)]): SnoozeInfo = copy(snoozes = f(snoozes))
}

5
app/models/User.scala Normal file
View File

@@ -0,0 +1,5 @@
package models
import com.mohiva.play.silhouette.api.Identity
case class User(username: String) extends Identity

View File

@@ -0,0 +1,20 @@
package models.odc
import models.odc.profile.api._
import slick.lifted.Tag
final case class CpeEntry(cpe: String, vendor: String, product: String)
class CpeEntries(tag: Tag) extends Table[(Int, CpeEntry)](tag, "cpeEntry") {
def id = column[Int]("id", O.PrimaryKey)
def cpe = column[String]("cpe")
def vendor = column[String]("vendor")
def product = column[String]("product")
def base = (cpe, vendor, product) <> (CpeEntry.tupled, CpeEntry.unapply)
def * = (id, base)
}

View File

@@ -0,0 +1,11 @@
package models.odc
import models.odc.profile.api._
final case class OdcProperty (id: String, value: String)
final class OdcProperties(tag: Tag) extends Table[OdcProperty](tag, "properties"){
def id = column[String]("id")
def value = column[String]("value")
def * = (id, value) <> (OdcProperty.tupled, OdcProperty.unapply)
}

View File

@@ -0,0 +1,17 @@
package models.odc
import com.ysoft.odc.Reference
import models.odc
import models.odc.profile.MappedJdbcType
import models.odc.profile.api._
import slick.lifted.Tag
class References (tag: Tag) extends Table[(Int, Reference)](tag, "reference") {
def cveId = column[Int]("cveid")
def name = column[String]("name")
def url = column[String]("url")
def source = column[String]("source")
def base = (source, url, name) <> (Reference.tupled, Reference.unapply)
def * = (cveId, base)
}

View File

@@ -0,0 +1,39 @@
package models.odc
import models.odc.profile.api._
import models.odc.profile.jdbcTypeFor
import slick.ast.TypedType
import models.odc.profile.MappedJdbcType
import slick.jdbc.JdbcType
import scala.reflect.ClassTag
// TODO: consider renaming to CpeEntryVulnerability or something like that
final case class SoftwareVulnerability (vulnerabilityId: Int, cpeEntryId: Int, includesAllPreviousVersionsRaw: Option[String]){
def includesAllPreviousVersions: Boolean = includesAllPreviousVersionsRaw match {
case Some("1") => true
case None => false
}
}
/*private class OdcBooleanType(implicit t: JdbcType[Option[String]]) extends MappedJdbcType[Boolean, Option[String]] {
override def map(t: Boolean): Option[String] = t match {
case true => Some("1")
case false => None
}
override def comap(u: Option[String]): Boolean = u match {
case Some("1") => true
case None => false
}
}*/
class SoftwareVulnerabilities(tag: Tag) extends Table[SoftwareVulnerability](tag, "software") {
def vulnerabilityId = column[Int]("cveid")
def cpeEntryId = column[Int]("cpeEntryId")
//private val bt = new OdcBooleanType()(jdbcTypeFor(implicitly[BaseColumnType[String]].optionType).asInstanceOf[JdbcType[Option[String]]])
//MappedJdbcType.base[Boolean, Option[String]](???, ???)(implicitly[ClassTag[Boolean]], )
def includesAllPreviousVersionsRaw = column[String]("previousVersion").?
def * = (vulnerabilityId, cpeEntryId, includesAllPreviousVersionsRaw) <> (SoftwareVulnerability.tupled, SoftwareVulnerability.unapply)
}

View File

@@ -0,0 +1,26 @@
package models.odc
import com.ysoft.odc.{CvssRating, CWE}
import models.odc.profile.api._
import slick.lifted.Tag
case class Vulnerability (cve: String, description: String, cweOption: Option[CWE], cvss: CvssRating)
class Vulnerabilities(tag: Tag) extends Table[(Int, Vulnerability)](tag, "vulnerability") {
def id = column[Int]("id")
def cve = column[String]("cve")
def description = column[String]("description")
def cweOption = column[String]("cwe").?
def cvssScore = column[Double]("cvssScore").?
def authentication = column[String]("cvssAuthentication").?
def availabilityImpact = column[String]("cvssAvailabilityImpact").?
def accessVector = column[String]("cvssAccessVector").?
def integrityImpact = column[String]("cvssIntegrityImpact").?
def cvssAccessComplexity = column[String]("cvssAccessComplexity").?
def cvssConfidentialityImpact = column[String]("cvssConfidentialityImpact").?
def cvssRating = (cvssScore, authentication, availabilityImpact, accessVector, integrityImpact, cvssAccessComplexity, cvssConfidentialityImpact) <> (CvssRating.tupled, CvssRating.unapply)
def cweOptionMapped = cweOption <> ((_: Option[String]).map(CWE.apply), (_: Option[CWE]).map(CWE.unapply))
def base = (cve, description, cweOptionMapped, cvssRating) <> (Vulnerability.tupled, Vulnerability.unapply)
def * = (id, base)
}

View File

@@ -0,0 +1,17 @@
package models
import slick.lifted.TableQuery
package object odc {
val profile = slick.driver.MySQLDriver
object tables {
val cpeEntries = TableQuery[CpeEntries]
val softwareVulnerabilities = TableQuery[SoftwareVulnerabilities]
val vulnerabilities = TableQuery[Vulnerabilities]
val references = TableQuery[References]
val properties = TableQuery[OdcProperties]
}
}

20
app/models/package.scala Normal file
View File

@@ -0,0 +1,20 @@
import slick.lifted.TableQuery
/**
* Created by user on 8/12/15.
*/
package object models {
val profile = slick.driver.PostgresDriver
val jodaSupport = com.github.tototoshi.slick.PostgresJodaSupport
object tables {
val libraries = TableQuery[Libraries]
val libraryTagAssignments = TableQuery[LibraryTagAssignments]
val tags = TableQuery[LibraryTags]
val snoozesTable = TableQuery[Snoozes]
val authTokens = TableQuery[CookieAuthenticators]
}
}

View File

@@ -0,0 +1,99 @@
package modules
import java.io._
import java.net.URLEncoder
import java.nio.file.{Files, Path, Paths}
import akka.util.ClassLoaderObjectInputStream
import com.ysoft.odc._
import controllers.MissingGavExclusions
import play.api.cache.CacheApi
import play.api.inject.{Binding, Module}
import play.api.{Configuration, Environment, Logger}
import scala.concurrent.duration.Duration
import scala.reflect.ClassTag
import scala.util.{Failure, Success, Try}
/**
* This class is rather a temporary hack and should be replaced by something better.
*
* Issues:
* * Thread safety
* * fsync: https://stackoverflow.com/questions/4072878/i-o-concept-flush-vs-sync
* * probably not removing files that are not used for a long time
* @param path
*/
class FileCacheApi(path: Path) extends CacheApi{
private def cacheFile(name: String) = path.resolve("X-"+URLEncoder.encode(name, "utf-8"))
override def remove(key: String): Unit = Files.deleteIfExists(cacheFile(key))
private def serialize(value: Any, duration: Duration) = {
val out = new ByteArrayOutputStream()
import com.github.nscala_time.time.Imports._
new ObjectOutputStream(out).writeObject((value, if(duration.isFinite()) Some(DateTime.now.plus(duration.toMillis)) else None))
out.toByteArray
}
private def unserialize[T](data: Array[Byte]): Try[T] = {
val in = new ByteArrayInputStream(data)
import com.github.nscala_time.time.Imports._
try{
new ClassLoaderObjectInputStream(this.getClass.getClassLoader, in).readObject() match {
case (value, None) => Success(value.asInstanceOf[T])
case (value, Some(exp: DateTime)) if exp < DateTime.now => Success(value.asInstanceOf[T])
case _ => Failure(new RuntimeException("cache expired"))
}
}catch{
case e: ObjectStreamException => Failure(e)
}
}
override def set(key: String, value: Any, expiration: Duration): Unit = {
Files.write(cacheFile(key), serialize(value, expiration))
}
override def get[T: ClassTag](key: String): Option[T] = {
val f = cacheFile(key)
if(Files.exists(f)){
val res = unserialize[T](Files.readAllBytes(f))
res match {
case Failure(e) =>
Logger.warn("not using cache for following key, removing that: "+key, e)
remove(key)
case Success(_) => // nothing to do
}
res.toOption
}else{
None
}
}
override def getOrElse[A: ClassTag](key: String, expiration: Duration)(orElse: => A): A = get(key).getOrElse{
val v = orElse
set(key, v, expiration)
v
}
}
class ConfigModule extends Module {
override def bindings(environment: Environment, configuration: Configuration): Seq[Binding[_]] = Seq(
bind[String].qualifiedWith("bamboo-server-url").toInstance(configuration.getString("yssdc.bamboo.url").getOrElse(sys.error("Key yssdc.bamboo.url is not set"))),
configuration.getString("yssdc.reports.provider") match{
case Some("bamboo") => bind[Downloader].to[BambooDownloader]
// not ready yet: case Some("files") => bind[Downloader].to[LocalFilesDownloader]
case other => sys.error(s"unknown provider: $other")
},
bind[MissingGavExclusions].qualifiedWith("missing-GAV-exclusions").toInstance(MissingGavExclusions(
configuration.getStringSeq("yssdc.exclusions.missingGAV.bySha1").getOrElse(Seq()).toSet.map(Exclusion))
)
) ++
configuration.getString("play.cache.path").map(cachePath => bind[CacheApi].toInstance(new FileCacheApi(Paths.get(cachePath)))) ++
configuration.getString("yssdc.reports.bamboo.sessionId").map{s => bind[BambooAuthentication].toInstance(new SessionIdBambooAuthentication(s))} ++
configuration.getString("yssdc.reports.bamboo.user").map{u => bind[BambooAuthentication].toInstance(new CredentialsBambooAuthentication(u, configuration.getString("yssdc.reports.bamboo.password").get))} ++
configuration.getString("yssdc.reports.path").map{s => bind[String].qualifiedWith("reports-path").toInstance(s)}
}

View File

@@ -0,0 +1,78 @@
package modules
import com.google.inject.{AbstractModule, Provides}
import com.mohiva.play.silhouette.api.repositories.AuthInfoRepository
import com.mohiva.play.silhouette.api.util._
import com.mohiva.play.silhouette.api.{Environment, EventBus}
import com.mohiva.play.silhouette.api.services.AuthenticatorService
import com.mohiva.play.silhouette.impl.authenticators.{CookieAuthenticatorService, CookieAuthenticatorSettings, CookieAuthenticator}
import com.mohiva.play.silhouette.impl.daos.DelegableAuthInfoDAO
import com.mohiva.play.silhouette.impl.providers.{CredentialsProvider, SocialProviderRegistry}
import com.mohiva.play.silhouette.impl.repositories.DelegableAuthInfoRepository
import com.mohiva.play.silhouette.impl.util.{BCryptPasswordHasher, SecureRandomIDGenerator, DefaultFingerprintGenerator}
import models.User
import net.codingwell.scalaguice.ScalaModule
import play.api.libs.ws.WSClient
import play.api.{Application, Configuration}
import services._
import net.ceedubs.ficus.Ficus._
import net.ceedubs.ficus.readers.ArbitraryTypeReader._
import play.api.libs.concurrent.Execution.Implicits._
class SilhouetteModule extends AbstractModule with ScalaModule{
override def configure(): Unit = {
bind[FingerprintGenerator].toInstance(new DefaultFingerprintGenerator(false))
bind[IDGenerator].toInstance(new SecureRandomIDGenerator())
bind[Clock].toInstance(Clock())
}
@Provides
def provideAuthInfoRepository(passwordInfoDAO: DelegableAuthInfoDAO[PasswordInfo]): AuthInfoRepository = {
new DelegableAuthInfoRepository(passwordInfoDAO)
}
@Provides
def provideAuthenticatorService(
fingerprintGenerator: FingerprintGenerator,
idGenerator: IDGenerator,
configuration: Configuration,
clock: Clock,
tokenService: TokenService
): AuthenticatorService[CookieAuthenticator] = {
val config = configuration.underlying.as[CookieAuthenticatorSettings]("silhouette.authenticator")
new CookieAuthenticatorService(
config,
Some(tokenService),
fingerprintGenerator,
idGenerator,
clock
)
}
@Provides
def provideCredentialsVerificationService(configuration: Configuration, app: Application)(implicit wSClient: WSClient): CredentialsVerificationService = {
configuration.underlying.as[String]("silhouette.credentialsVerificationService.type") match {
case "allow-all" => new AllowAllCredentialsVerificationService(app)
case "external" => new ExternalCredentialsVerificationService(configuration.underlying.as[String]("silhouette.credentialsVerificationService.url"))
}
}
@Provides
def provide(userService: UserService): DelegableAuthInfoDAO[PasswordInfo] = userService
@Provides
def provideEnvironment(
userService: UserService,
authenticatorService: AuthenticatorService[CookieAuthenticator],
eventBus: EventBus): Environment[User, CookieAuthenticator] = {
Environment[User, CookieAuthenticator](
userService,
authenticatorService,
Seq(),
eventBus
)
}
}

View File

@@ -0,0 +1,13 @@
package services
import scala.concurrent.Future
class AllowAllCredentialsVerificationService(app: play.api.Application) extends CredentialsVerificationService{
if(app.mode != play.api.Mode.Dev){
// safety check:
sys.error("allow-all can be used in dev mode only")
}
override def verifyCredentials(username: String, password: String): Future[Boolean] = Future.successful(true)
}

View File

@@ -0,0 +1,7 @@
package services
import scala.concurrent.Future
trait CredentialsVerificationService {
def verifyCredentials(username: String, password: String): Future[Boolean]
}

View File

@@ -0,0 +1,17 @@
package services
import play.api.libs.json.Json
import play.api.libs.ws.{WS, WSClient}
import scala.concurrent.{Future, ExecutionContext}
class ExternalCredentialsVerificationService(url: String)(implicit executionContext: ExecutionContext, wSClient: WSClient) extends CredentialsVerificationService{
override def verifyCredentials(username: String, password: String): Future[Boolean] = {
WS.clientUrl(url).post(Json.toJson(Map("username" -> username, "password" -> password))).map{ response =>
response.body match {
case "OK" => true
case "bad" => false
}
}
}
}

View File

@@ -0,0 +1,78 @@
package services
import com.google.inject.Inject
import models._
import models.tables._
import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider}
import scala.concurrent.{ExecutionContext, Future}
class LibrariesService @Inject() (protected val dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends HasDatabaseConfigProvider[models.profile.type]{
import dbConfig.driver.api._
def all: Future[Seq[(Int, Library)]] = db.run(libraries.result)
def allBase: Future[Seq[Library]] = db.run(libraries.map(_.base).result)
// TODO: unify or differentiate librariesForTags and byTags
def librariesForTags(tagIds: Iterable[Int]): Future[Seq[(Int, (Int, Library))]] = db.run(
libraryTagAssignments
.join(libraries).on { case (a, l) => a.libraryId === l.id }
.filter { case (a, l) => a.libraryTagId inSet tagIds }
.map { case (a, l) => a.libraryTagId -> l }
.result
)
def byTags(libraryIds: Set[Int]) = {
(db.run(
libraryTagAssignments
.filter(_.libraryId inSet libraryIds)
.result
): Future[Seq[LibraryTagAssignment]]).map(_.groupBy(_.libraryId).mapValues(_.toSet).map(identity))
}
def setClassified(libraryId: Int, classified: Boolean): Future[_] = db.run(libraries.filter(_.id === libraryId).map(_.classified).update(classified))
def touched(tagIds: Iterable[Int]): Future[Seq[(Int, Library)]] = db.run(libraries.filter(l => l.classified || l.id.inSet(tagIds) ).result)
def unclassified: Future[Seq[(Int, Library)]] = db.run(libraries.filter(!_.classified).sortBy(l => l.plainLibraryIdentifierUnmapped).result)
def insert(lib: Library) = db.run(libraries.map(_.base).returning(libraries.map(_.id)) += lib)
def insertMany(newItems: Iterable[Library]): Future[_] = db.run(libraries.map(_.base) ++= newItems)
def filtered(requiredClassification: Option[Boolean], requiredTagsOption: Option[Set[Int]]): Query[Libraries, (Int, Library), Seq] = {
libraries
.joinLeft(libraryTagAssignments).on { case (l, a) => l.id === a.libraryId }
.filter { case (l, a) => requiredClassification.map(l.classified === _).getOrElse(l.classified === l.classified) } // classification matches
.filter { case (l, a) =>
requiredTagsOption.fold(
// actually a.isEmpty; The t.isEmpty should work, but there is a bug it uses (… = 1) on a null value, which has a different semantics in SQL. Related to https://github.com/slick/slick/issues/1156 .
// So, I created following workaround:
a.fold(true.asColumnOf[Boolean])(_ => false) // Filter only libraries with no tags (i.e. LEFT JOIN added NULLs (they corresponsd to None value))
)(requiredTagsSet =>
if (requiredTagsSet.isEmpty) true.asColumnOf[Boolean] // If we don't filter any by tag, we should allow all
else a.map(_.libraryTagId inSet requiredTagsSet).getOrElse(false.asColumnOf[Boolean]) // Filter tags
)
}
.groupBy { case (l, a) => l } // a library with multiple tags should be present only once
.map { case (l, q) => (l, q.size) } // we are not interested in the tags, but only in their count
.filter { case (l, c) => requiredTagsOption.fold( true.asColumnOf[Boolean] )(requiredTagsSet => c >= requiredTagsSet.size ) } // filter libraries with all the tags we are looking for
.map { case (l, c) => l } // all is filtered, so we need libraries only
.sortBy { l => l.plainLibraryIdentifierUnmapped }
}
def byPlainLibraryIdentifiers(plainLibraryIdentifiers: Set[PlainLibraryIdentifier]): Future[Map[PlainLibraryIdentifier, (Int, Library)]] = {
val groupedIdentifiers = plainLibraryIdentifiers.groupBy(_.libraryType).mapValues(_.map(_.libraryIdentifier)).map(identity)
val resFuture: Future[Seq[(Int, Library)]] = db.run(libraries.filter{l =>
val conditions = for((libraryType, identifiers) <- groupedIdentifiers) yield l.libraryType === libraryType && l.libraryIdentifier.inSet(identifiers)
conditions.foldLeft(false.asColumnOf[Boolean])(_ || _)
}.result)
resFuture.map(
//_.toSet.groupBy(_._2.plainLibraryIdentifier)
_.map(x => x._2.plainLibraryIdentifier -> x).toMap
)
}
}

View File

@@ -0,0 +1,34 @@
package services
import com.google.inject.Inject
import models.tables._
import models.{LibraryTagAssignment, LibraryTagPair}
import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider}
import scala.concurrent.{ExecutionContext, Future}
class LibraryTagAssignmentsService @Inject() (protected val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[models.profile.type]{
import dbConfig.driver.api._
def all = db.run(libraryTagAssignments.result): Future[Seq[LibraryTagAssignment]]
def insert(item: LibraryTagAssignment) = db.run(libraryTagAssignments += item)
def remove(libraryTagPair: LibraryTagPair) = db.run(
libraryTagAssignments
.filter(_.libraryTagId === libraryTagPair.tagId)
.filter(_.libraryId === libraryTagPair.libraryId)
.delete
)
def forLibraries(libraryIds: Set[Int]): Future[Seq[LibraryTagAssignment]] = db.run(libraryTagAssignments.filter(_.libraryId inSet libraryIds).result)
def byLibrary(implicit executionContext: ExecutionContext) = all.map(_.groupBy(_.libraryId).withDefaultValue(Seq()))
def tagsToLibraries(tagAssignmentsFuture: Future[Seq[LibraryTagAssignment]])(implicit executionContext: ExecutionContext): Future[Map[Int, Set[LibraryTagAssignment]]] =
tagAssignmentsFuture.map(x => tagsToLibraries(x))
def tagsToLibraries(tagAssignments: Seq[LibraryTagAssignment]): Map[Int, Set[LibraryTagAssignment]] = tagAssignments.groupBy(_.tagId).mapValues(_.toSet).map(identity).withDefaultValue(Set.empty)
}

View File

@@ -0,0 +1,124 @@
package services
import java.lang.{Boolean => JBoolean}
import java.util.{Map => JMap}
import _root_.org.owasp.dependencycheck.data.nvdcve.CveDB
import _root_.org.owasp.dependencycheck.dependency.VulnerableSoftware
import _root_.org.owasp.dependencycheck.utils.{DependencyVersion, DependencyVersionUtil, Settings}
import com.github.nscala_time.time.Imports._
import com.google.inject.Inject
import models.odc.OdcProperty
import models.odc.tables._
import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider}
import play.db.NamedDatabase
import scala.concurrent.{ExecutionContext, Future}
class OdcService @Inject()(@NamedDatabase("odc") protected val dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends HasDatabaseConfigProvider[models.odc.profile.type]{
import dbConfig.driver.api._
def getVulnerableSoftware(id: Int): Future[Seq[com.ysoft.odc.VulnerableSoftware]] = {
db.run(softwareVulnerabilities.joinLeft(cpeEntries).on((sv, ce) => sv.cpeEntryId === ce.id).filter{case (sv, ceo) => sv.vulnerabilityId === id}.result).map{rawRefs =>
rawRefs.map{
case (softVuln, Some((_, cpeEntry))) => com.ysoft.odc.VulnerableSoftware(allPreviousVersion = softVuln.includesAllPreviousVersions, name=cpeEntry.cpe)
}
}
}
def getReferences(id: Int): Future[Seq[com.ysoft.odc.Reference]] = db.run(references.filter(_.cveId === id).map(_.base).result)
def getVulnerabilityDetails(id: Int): Future[Option[com.ysoft.odc.Vulnerability]] = {
for {
bareVulnOption <- db.run(vulnerabilities.filter(_.id === id).map(_.base).result).map(_.headOption)
vulnerableSoftware <- getVulnerableSoftware(id)
references <- getReferences(id)
} yield bareVulnOption.map{bareVuln =>
com.ysoft.odc.Vulnerability(
name = bareVuln.cve,
cweOption = bareVuln.cweOption,
cvss = bareVuln.cvss,
description = bareVuln.description,
vulnerableSoftware = vulnerableSoftware,
references = references
)
}
}
private def parseCpe(cpe: String) = {
val sw = new VulnerableSoftware()
sw.parseName(cpe)
sw
}
def parseVersion(version: String): DependencyVersion = {
DependencyVersionUtil.parseVersion(version)
}
/*def parseCpeVersion(cpe: String): DependencyVersion = { // strongly inspired by org.owasp.dependencycheck.data.nvdcve.CveDB.parseDependencyVersion(cpe: VulnerableSoftware): DependencyVersion
def StringOption(s: String) = Option(s).filterNot(_.isEmpty)
val sw = parseCpe(cpe)
StringOption(sw.getVersion) match {
case None ⇒ new DependencyVersion("-")
case Some(bareVersionString) ⇒
DependencyVersionUtil.parseVersion(
StringOption(sw.getUpdate) match {
case None ⇒ bareVersionString
case Some(update) ⇒ s"$bareVersionString.$update"
}
)
}
}*/
def findRelevantCpes(versionlessCpe: String, version: String) = {
println(s"versionlessCpe: $versionlessCpe")
val Seq("cpe", "/a", vendor, product, rest @ _*) = versionlessCpe.split(':').toSeq
val cpesFuture = db.run(
cpeEntries.filter(c =>
c.vendor === vendor && c.product === product
).result
)
for(cpes <- cpesFuture){println(s"cpes: $cpes")}
val cpesMapFuture = cpesFuture.map(_.toMap)
val cpeIdsFuture = cpesFuture.map(_.map(_._1))
val parsedVersion = parseVersion(version)
val res = for{
cpeIds <- cpeIdsFuture
relevantVulnerabilities <- db.run(
softwareVulnerabilities.join(vulnerabilities).on( (sv, v) => sv.vulnerabilityId === v.id)
.filter{case (sv, v) => sv.cpeEntryId inSet cpeIds}.map{case (sv, v) sv}.result
).map(_.groupBy(_.vulnerabilityId).mapValues(_.toSet))
cpesMap <- cpesMapFuture
//relevantVulnerabilities <- db.run(vulnerabilities.filter(_.id inSet relevantVulnerabilityIds).result)
} yield relevantVulnerabilities.filter{case (vulnId, sv) => Option(CveDbHelper.matchSofware(
vulnerableSoftware = sv.map(sv => cpesMap(sv.cpeEntryId).cpe -> sv.includesAllPreviousVersions).toMap,
vendor = vendor,
product = product,
identifiedVersion = parsedVersion
)).isDefined}
res.map(_.values.toSet.flatten)
}
def loadUpdateProperties(): Future[Map[String, Long]] = db.run(properties.filter(_.id like "NVD CVE%").result).map(_.map{case OdcProperty(id, value) => (id, value.toLong)}.toMap)
def loadLastDbUpdate(): Future[DateTime] = loadUpdateProperties().map(vals => new DateTime(vals.values.max)) // TODO: timezone (I don't care much, though)
}
private[services] object CveDbHelper {
def matchSofware(vulnerableSoftware: Map[String, Boolean], vendor: String, product: String, identifiedVersion: DependencyVersion) = {
if(Settings.getInstance() == null){
Settings.initialize()// Initiallize ODC environment on first use; Needed for each thread.
}
val cd = new CveDB()
import scala.collection.JavaConversions._
val method = cd.getClass.getDeclaredMethod("getMatchingSoftware", classOf[JMap[String, JBoolean]], classOf[String], classOf[String], classOf[DependencyVersion])
method.setAccessible(true)
method.invoke(cd, mapAsJavaMap(vulnerableSoftware).asInstanceOf[JMap[String, JBoolean]], vendor, product, identifiedVersion)
}
}

View File

@@ -0,0 +1,19 @@
package services
import com.google.inject.Inject
import models._
import models.tables._
import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider}
import scala.concurrent.{ExecutionContext, Future}
class TagsService @Inject() (protected val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[models.profile.type]{
import dbConfig.driver.api._
def all: Future[Seq[(Int, LibraryTag)]] = db.run(tags.result)
def insertMany(newTags: Iterable[LibraryTag]): Future[_] = db.run(tags.map(_.base) ++= newTags)
def getById(id: Int)(implicit executionContext: ExecutionContext): Future[(Int, LibraryTag)] = db.run(tags.filter(_.id === id).result).map(_.head)
}

View File

@@ -0,0 +1,36 @@
package services
import javax.inject.Inject
import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator
import com.mohiva.play.silhouette.impl.daos.AuthenticatorDAO
import play.api.db.slick.{HasDatabaseConfigProvider, DatabaseConfigProvider}
import models.tables._
import scala.concurrent.{Future, ExecutionContext}
class TokenService @Inject() (protected val dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext)
extends AuthenticatorDAO[CookieAuthenticator]
with HasDatabaseConfigProvider[models.profile.type]{
import dbConfig.driver.api._
println(authTokens.schema.create.statements.toIndexedSeq)
override def find(id: String): Future[Option[CookieAuthenticator]] = {
db.run(authTokens.filter(_.id === id).result).map{_.headOption}
}
override def add(authenticator: CookieAuthenticator): Future[CookieAuthenticator] = {
db.run(authTokens += authenticator).map(_ => authenticator)
}
override def remove(id: String): Future[Unit] = {
db.run(authTokens.filter(_.id === id).delete).map(_ => ())
}
override def update(authenticator: CookieAuthenticator): Future[CookieAuthenticator] = {
db.run(authTokens.filter(_.id === authenticator.id).update(authenticator)).map(_ => authenticator)
}
}

View File

@@ -0,0 +1,29 @@
package services
import com.mohiva.play.silhouette.api.LoginInfo
import com.mohiva.play.silhouette.api.services.IdentityService
import com.mohiva.play.silhouette.api.util.PasswordInfo
import com.mohiva.play.silhouette.impl.daos.DelegableAuthInfoDAO
import models.User
import scala.concurrent.Future
class UserService extends DelegableAuthInfoDAO[PasswordInfo] with IdentityService[User]
{
override def retrieve(loginInfo: LoginInfo): Future[Option[User]] = if(loginInfo.providerID == "credentials-verification") Future.successful(Some(User(loginInfo.providerKey))) else Future.successful(None)
override def find(loginInfo: LoginInfo): Future[Option[PasswordInfo]] = {
println(s"loginInfo: $loginInfo")
???
}
override def update(loginInfo: LoginInfo, authInfo: PasswordInfo): Future[PasswordInfo] = ???
override def remove(loginInfo: LoginInfo): Future[Unit] = ???
override def save(loginInfo: LoginInfo, authInfo: PasswordInfo): Future[PasswordInfo] = ???
override def add(loginInfo: LoginInfo, authInfo: PasswordInfo): Future[PasswordInfo] = ???
}

View File

@@ -0,0 +1,12 @@
@import helper._
@(loginForm: Form[LoginRequest]/*, socialProviderRegistry: SocialProviderRegistry*/)(implicit requestHeader: DefaultRequest, messages: Messages)
@main("Log in"){
@form(routes.AuthController.authenticate()){
@CSRF.formField
@inputText(loginForm("username"))
@inputPassword(loginForm("password"))
@* checkbox(loginForm("rememberMe")) *@
<button type="submit">Log me in!</button>
}
}

View File

@@ -0,0 +1,18 @@
@(list: Traversable[_], name: String, id: String, collapse: Boolean = false, allowSnoozes: Boolean = true, versions: Map[String, Int])(content: => Html)(implicit rh: DefaultRequest, snoozes: SnoozesInfo, messages: Messages)
@if(list.nonEmpty){
@defining(snoozes(id)){ case si =>
@if(allowSnoozes) {
@snoozeButton(id, si, collapse)
}
<h2 id="@id" @if(si.isSnoozed){class="text-muted"} data-toggle="collapse" data-target="#@id-div, #@id-snooze-button">@if(si.isSnoozed){&#128564;} @name (@{list.size})</h2>
<div id="@id-div" class="collapse@if(!si.shouldCollapse(collapse)){ in}">
@if(allowSnoozes) {
@snoozeForm(id, si, versions)
}
@content
@if(allowSnoozes) {
@snoozesList(id, si, versions)
}
</div>
}
}

View File

@@ -0,0 +1,40 @@
@(
selectedDependencies: Seq[(Int, Library)],
allTags: Seq[(Int, LibraryTag)],
dependencyTags: Map[Int, Set[LibraryTagAssignment]],
requiredClassification: Option[Boolean],
requiredTagSet: Set[Int],
noTag: Boolean,
tagsLink: Set[Int] => Call,
classificationLink: Option[Boolean] => Call,
noTagLink: Boolean => Call
)(implicit header: DefaultRequest)
@main(s"${requiredClassification match{case Some(true) => "Classified" case Some(false) => "Unclassified" case None => "All"}} dependencies (${selectedDependencies.size})") {
<div>
<div class="btn-group">
@for((newClassification, name) <- Seq(None -> "All", Some(true) -> "Classified", Some(false) -> "Unclassified"); isCurrent = newClassification == requiredClassification){
<a class="btn @if(isCurrent){active} btn-default" href="@classificationLink(newClassification)">@name</a>
}
</div>
</div>
<a href="@noTagLink(!noTag)" class="btn btn-primary @if(!noTag){active}">Required tags:</a>
@if(!noTag) {
@for((tagId, tag) <- allTags.sortBy(_._2.name); enabled = requiredTagSet contains tagId) {
<a
href="@tagsLink(if(enabled) requiredTagSet - tagId else requiredTagSet + tagId)"
class="btn btn-default @if(requiredTagSet contains tagId) {active btn-success}"
title="@tag.note"
>@tag.name</a>
}
}
<hr>
@dependencyClassification(
prefix = "dependency",
dependencies = selectedDependencies,
allTags = allTags,
dependenciesTags = dependencyTags,
details = (_, _) => {
Html("")
}
)
}

View File

@@ -0,0 +1,40 @@
@(
prefix: String,
dependencies: Seq[(Int, Library)],
allTags: Seq[(Int, LibraryTag)],
dependenciesTags: Map[Int, Set[LibraryTagAssignment]],
details: (Int, PlainLibraryIdentifier) => Html
)
<ul class="table">
@for((libraryId, Library(lib, classified)) <- dependencies){
<li class="versionless-dependency">
<a href="#@prefix-@libraryId" data-toggle="collapse">@lib.libraryType: @lib.libraryIdentifier</a>
<span class="related-links">
@if(lib.libraryType == LibraryType.Maven){
<a class="text-muted" href="http://mvnrepository.com/artifact/@lib.libraryIdentifier.replaceFirst(":", "/")">mvnrepository.com»</a>
<a class="text-muted" href="https://libraries.io/maven/@lib.libraryIdentifier">libraries.io»</a>
@defining(lib.libraryIdentifier.takeWhile(_ != ':').split('.')){ reverseDomain =>
@if(! reverseDomain.startsWith(Seq("javax")) ){
@for(i <- reverseDomain.length to 2 by -1; guessedDomain = reverseDomain.take(i).reverse.mkString("."); if !(TooGenericDomains contains guessedDomain)){
<a class="text-muted" href="http://@guessedDomain">@guessedDomain»</a>
}
}
}
}
<a class="text-muted" href="https://www.google.com/search?q=@helper.urlEncode(lib.libraryIdentifier)&ie=utf-8&oe=utf-8">Google»</a>
</span>
<div id="@prefix-@libraryId" class="collapse">
@details(libraryId, lib)
@defining(dependenciesTags.getOrElse(libraryId, Set.empty)) { libraryTags =>
@for(
(tagId, tag) <- allTags.sortBy(_._2.name.toLowerCase);
exists = libraryTags.map(_.tagId) contains tagId
){
<button class="btn btn-default@if(exists){ btn-success}" onclick="toggleTag(this)" data-library-id="@libraryId" data-tag-id="@tagId" title="@tag.note">@tag.name</button>
}
}
<button class="btn btn-default@if(classified){ btn-success}" onclick="toggleClassified(this)" data-library-id="@libraryId"></button>
</div>
</li>
}
</ul>

View File

@@ -0,0 +1,77 @@
@(idPrefix: String, list: Seq[GroupedDependency], selectorOption: Option[String], expandByDefault: Boolean = true, addButtons: Boolean = true)
@cpeHtmlId(cpe: String) = @{
cpe.getBytes("utf-8").mkString("-")
}
@for(dep <- list; depPrefix = s"$idPrefix-${dep.sha1}"){
<h3 class="library-identification" id="@depPrefix-head" data-toggle="collapse" data-target="#@depPrefix-details">
@libraryIdentification(dep, Some(cpe => s"$idPrefix-${dep.sha1}-suppression-cpe-${cpeHtmlId(cpe)}"), addLink = false, addButtons = addButtons)
@for(s <- dep.maxCvssScore) {
<span class="severity">
(<span title="highest vulnerability score" class="explained">@s</span>
× <span class="explained" title="affected project count">@dep.projects.size</span>
= <span class="explained score" title="total score">@dep.ysdssScore</span>)
(vulns:&nbsp;@dep.vulnerabilities.size)
</span>
}
@dep.cpeIdentifiers.toSeq match {
case Seq() => {}
case cpeIds => {
<a href="@routes.Statistics.searchVulnerableSoftware(cpeIds.map(_.name.split(':').take(4).mkString(":")).toSeq, None)" title="Search for known vulnerabilities"><span class="glyphicon glyphicon-flash"></span></a>
}
}
</h3>
@for(identifier <- dep.identifiers; cpe <- identifier.toCpeIdentifierOption ) {
<div id="@(s"$idPrefix-${dep.sha1}-suppression-cpe-${cpeHtmlId(cpe)}")" class="collapse">@SuppressionXml.forCpe(dep, cpe)</div>
}
<div id="@depPrefix-details" class="collapse @if(expandByDefault){ in }">
@if(dep.descriptions.size > 1){
<div class="alert alert-warning">Multiple descriptions for this dependency!</div>
}
@for(descriptionParagraphs <- dep.parsedDescriptions){
<div class="description">
@for(descriptionParagraphLines <- descriptionParagraphs){
<p>
@for(line <- descriptionParagraphLines) {
@line<br>
}
</p>
}
</div>
}
<h4 data-toggle="collapse" data-target="#@depPrefix-evidence-details">Evidence</h4>
<table id="@depPrefix-evidence-details" class="collapse table table-bordered table-condensed">
<tr>
<th>confidence</th>
<th>evidence type</th>
<th>name</th>
<th>source</th>
<th>value</th>
</tr>
@for(ev <- dep.dependencies.keySet.map(_.evidenceCollected).flatten){
<tr>
<td>@ev.confidence
<td>@ev.evidenceType
<td>@ev.name
<td>@ev.source
<td>@ev.value
</tr>
}
</table>
<h4 data-toggle="collapse" data-target="#@depPrefix-projects-details">Affected projects (@dep.projects.size)</h4>
<ul id="@depPrefix-projects-details" class="collapse in">@for(p <- dep.projects.toIndexedSeq.sorted){<li>@friendlyProjectName(p)</li>}</ul>
<h4 data-toggle="collapse" data-target="#@depPrefix-vulnerabilities-details">Vulnerabilities (@dep.vulnerabilities.size)</h4>
<ul id="@depPrefix-vulnerabilities-details" class="collapse in">
@for(vuln <- dep.vulnerabilities.toSeq.sortBy(_.cvssScore.map(-_)); vulnPrefix = s"$depPrefix-vulnerabilities-details-${vuln.name}"){
<li>
<h5 data-toggle="collapse" data-target="#@vulnPrefix-details">@vuln.name <a href="@routes.Statistics.vulnerability(vuln.name, selectorOption)"><span class="glyphicon glyphicon-log-out"></span></a></h5>
<div id="@vulnPrefix-details" class="collapse">
@vulnerability("h6", s"$idPrefix-${dep.sha1}", vuln)
<h6 data-toggle="collapse" data-target="#@(s"$idPrefix-${dep.sha1}-suppression-cve-${vuln.name}")">CVE suppression</h6>
<div id="@(s"$idPrefix-${dep.sha1}-suppression-cve-${vuln.name}")" class="collapse">@SuppressionXml.forVuln(dep, vuln)</div>
</div>
</li>
}
</ul>
</div>
}

View File

@@ -0,0 +1 @@
<b>all</b>

View File

@@ -0,0 +1,2 @@
@(currentProject: ReportInfo)
Project: <b>@friendlyProjectName(currentProject)</b>

View File

@@ -0,0 +1,2 @@
@(teamId: String)
Team: <b>@teamId</b>

View File

@@ -0,0 +1,3 @@
@(idPrefix: String)(ht: String)(name: String, description: String)(content: Html)
<@ht id="@idPrefix-@name-header" data-toggle="collapse" data-target="#@idPrefix-@name-details">@description</@ht>
<div id="@idPrefix-@name-details" class="collapse in">@content</div>

View File

@@ -0,0 +1,12 @@
@(name: String, id: String, collapse: Boolean = false, allowSnoozes: Boolean = true, versions: Map[String, Int])(list: Seq[GroupedDependency])(implicit rh: DefaultRequest, snoozes: SnoozesInfo, messages: Messages)
@conditionalList(list, name, id, collapse = collapse, allowSnoozes = allowSnoozes, versions = versions){
<table class="table">
@for(dep <- list){
<tr>
<td>@identifiers(dep.mavenIdentifiers)</td>
<td>@identifiers(dep.cpeIdentifiers)</td>
<td>@dep.descriptions</td>
</tr>
}
</table>
}

View File

@@ -0,0 +1,10 @@
package views.html
import com.ysoft.odc.{GroupedDependency, Vulnerability}
object SuppressionXml {
def forCpe(dep: GroupedDependency, cpe: String) = suppressionXmlPre(dep, <cpe>{cpe}</cpe>)
def forVuln(dep: GroupedDependency, vuln: Vulnerability) = suppressionXmlPre(dep, <cve>{vuln.name}</cve>)
}

View File

@@ -0,0 +1,9 @@
package views
import models.User
package object html {
type SortedMap[A, B] = scala.collection.SortedMap[A, B]
type UserAwareRequest[T] = controllers.AuthenticatedController#UserAwareRequest[T]
type DefaultRequest = UserAwareRequest[_]
}

View File

@@ -0,0 +1,3 @@
@(identifier: Identifier, addLink: Boolean = true)
@identifier.confidence.toString.toLowerCase:
@secureLink(if(addLink) identifier.url else ""){@identifier.name}

View File

@@ -0,0 +1,7 @@
@(identifiers: Set[Identifier])
@for(i <- identifiers){
<div class="identifier">
@identifier(i)
</div>
}

View File

@@ -0,0 +1,88 @@
@(
vulnerableDependencies: Seq[GroupedDependency],
unclassifiedDependencies: Seq[(Int, Library)],
warnings: Seq[Warning],
groupedDependencies: Seq[GroupedDependency],
dependenciesForLibraries: Map[PlainLibraryIdentifier, Set[GroupedDependency]],
allTags: Seq[(Int, LibraryTag)],
relatedDependenciesTags: Map[Int, Set[LibraryTagAssignment]],
librariesForTagsWithWarning: SortedMap[(Int, LibraryTag), Seq[(Int, Library)]],
lastRefreshTime: DateTime,
versions: Map[String, Int]
)(implicit req: DefaultRequest, snoozes: SnoozesInfo , messages: Messages)
@import com.ysoft.odc.Confidence
@import helper._
@main("Y Soft Dependency status"){
@form(routes.Application.purgeCache(versions, "index")){
@CSRF.formField
<button type="submit" class="btn btn-default">Purge cache</button> <span class="text-muted">(last update form build server: @lastRefreshTime)</span>
}
@conditionalList(warnings, "Warnings", "warnings", allowSnoozes = false, versions = versions){
@for(w <- warnings.sortBy(w => (-w.severity.id, w.id)); isLow <- Some(w.severity < WarningSeverity.Warning) ){
@defining(snoozes(s"warning-${w.id}")){ si =>
<div class="alert @(w.severity match {
case controllers.WarningSeverity.Error => "alert-danger"
case controllers.WarningSeverity.Warning => "alert-warning"
case controllers.WarningSeverity.Info => "alert-info"
}) @if(si.isSnoozed){ text-muted}" id="warning-@w.id">
<button data-toggle="collapse" class="btn btn-sm toggle-warning" data-target="#warning-@w.id-details, #warning-@w.id-snooze-button">
<span class="glyphicon glyphicon-grain" aria-hidden="true"></span>
</button>
@snoozeButton(s"warning-${w.id}", si, collapseByDefault = false)
@if(w.allowSnoozes){
@snoozeForm(s"warning-${w.id}", si, versions)
}
<div id="warning-@w.id-details" class="collapse @if(!si.shouldCollapse(default = isLow)){in}">
@w.html
@if(w.allowSnoozes){
@snoozesList(s"warning-${w.id}", si, versions)
}
</div>
<div class="clearfix"></div>
</div>
}
}
}
@conditionalList(vulnerableDependencies, s"Vulnerable dependencies ${(Seq(None) ++ vulnerableDependencies.map(_.maxCvssScore)).max.map{maxScore => s"(max CVSS: $maxScore)"}.getOrElse("")}", "vulnerable", versions = versions) {
@dependencyList("vulnerable", vulnerableDependencies.sortBy(d => (d.maxCvssScore.map(-_), d.cpeIdentifiers.map(_.toCpeIdentifierOption.get).toSeq.sorted.mkString(" "))), None)
}
@* groupedDependencyList("Unclassified dependencies", "unclassified")(unclassifiedDependencies) *@
@conditionalList(unclassifiedDependencies, "Unclassified dependencies", "unclassified", versions = versions){
@dependencyClassification(
prefix = "unclassified-dependency",
dependencies = unclassifiedDependencies,
allTags = allTags,
dependenciesTags = relatedDependenciesTags,
details = (libraryId: Int, lib: PlainLibraryIdentifier) => {
dependenciesForLibraries.get(lib).fold{Html("<p>No details</p>")} { deps =>
dependencyList(s"unclassified-library-$libraryId-details", deps.toSeq /*TODO: sort */, None)
}
}
)
}
@groupedDependencyList("Dependencies with low confidence of GAV", "gav-low-confidence", versions = versions)(groupedDependencies.filter(_.mavenIdentifiers.exists(_.confidence < Confidence.High)))
@for(((tagId, tag), libraries) <- librariesForTagsWithWarning){
@conditionalList(libraries, s"${tag.name}", s"tag-warning-$tagId", versions = versions){
@for(note <- tag.note){
<p>@note</p>
}
@dependencyClassification(
prefix = s"tag-warning-$tagId-list",
dependencies = libraries,
allTags = allTags,
dependenciesTags = relatedDependenciesTags,
details = (_, _) => Html("")
)
}
}
@* @groupedDependencyList("All dependencies", "all", collapse = true)(groupedDependencies) *@
}

View File

@@ -0,0 +1,16 @@
@(dep: GroupedDependency, suppressionXmlIdOption: Option[String => String] = None, addLink: Boolean = true, addButtons: Boolean = true)
@import com.ysoft.odc.Confidence
@import scala.math.Ordered.orderingToOrdered
@if(!dep.identifiers.exists(_.confidence >= Confidence.High)){
<span class="badge">file: @dep.fileNames.toSeq.sorted.mkString(", ")@if(addButtons){<span class="btn-xs library-identification-badge-hack">&nbsp;</span>}</span>
}
@for(id <- dep.identifiers.toSeq.sortBy(i => (i.confidence, i.identifierType, i.name, i.url)).reverse){
<span class="badge">
@identifier(id, addLink)
@for(cpe <- id.toCpeIdentifierOption; suppressionXmlId <- suppressionXmlIdOption; if addButtons){
<button class="btn btn-default btn-xs" data-toggle="collapse" data-target="#@suppressionXmlId(cpe)">×</button>
}
@if(addButtons && suppressionXmlIdOption.isDefined){<span class="btn-xs library-identification-badge-hack">&nbsp;</span>}
</span>
}

96
app/views/main.scala.html Normal file
View File

@@ -0,0 +1,96 @@
@import helper._
@(title: String, headExtension: Html = Html(""), projectsOption: Option[(ProjectsWithSelection, Option[String] => Call)] = None)(content: Html)(implicit header: DefaultRequest)
<!DOCTYPE html>
<html>
<head>
<title>@title</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" type="text/css" href="@routes.Assets.versioned("lib/bootstrap/css/bootstrap.css")">
<link rel="stylesheet" type="text/css" href="@routes.Assets.versioned("lib/bootstrap-datepicker/css/bootstrap-datepicker3.css")">
<link rel="stylesheet" type="text/css" href="@routes.Assets.versioned("css/main.css")">
<script type="text/javascript" src="@routes.Assets.versioned("lib/jquery/jquery.js")"></script>
<script type="text/javascript" src="@routes.Assets.versioned("lib/bootstrap/js/bootstrap.js")"></script>
<script type="text/javascript" src="@routes.Assets.versioned("lib/bootstrap-datepicker/js/bootstrap-datepicker.js")"></script>
<script type="text/javascript" src="@routes.Application.javascriptRoutes"></script>
<script type="text/javascript" src="@routes.Assets.versioned("js/main.js")"></script>
<script type="text/javascript">
if(!window.Routes){
window.Routes = {};
}
window.Routes.addTag = "@routes.Application.addTag";
window.Routes.removeTag = "@routes.Application.removeTag";
@* window.Routes.markClassified = "@routes.Application.markClassified"; *@
@* window.Routes.markUnclassified = "@routes.Application.markUnlassified"; *@
</script>
@if(!header.secure){
<script async defer type="text/javascript" src="@(routes.Application.testHttps(header.method == "GET").absoluteURL(secure = true))"></script>
}
@headExtension
</head>
<body>
<nav class="navbar navbar-inverse navbar-fixed-top" id="header">
<div class="container">
<div class="navbar-header">
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
@* <a class="navbar-brand" href="#">YSSDC</a> *@
</div>
<div id="navbar" class="collapse navbar-collapse">
<ul class="nav navbar-nav">
<li><a href="@routes.Application.index(Map())">Status</a></li>
<li><a href="@routes.Application.dependencies(None)">Tags</a></li>
<li><a href="@routes.Statistics.basic(None)">Tag statistics</a></li>
<li><a href="@routes.Statistics.vulnerabilities(None, None)">Vulnerabilities</a></li>
<li><a href="@routes.Statistics.vulnerableLibraries(None)">Vulnerable libraries</a></li>
<li>
@for((ProjectsWithSelection(filter, projects, teams), link) <- projectsOption){
<div id="project-selector">
<div class="dropdown">
<button class="btn @if(filter.filters){btn-warning}else{btn-primary} dropdown-toggle" type="button" data-toggle="dropdown">@filter.descriptionHtml
<span class="caret"></span>
</button>
<ul class="dropdown-menu">
<li><a href="@link(None)"><i>all projects</i></a></li>
<li class="base-project"><a href="#">Teams</a></li>
@for(team <- teams){
<li><a href="@link(Some("team:"+team.id))" title="team leader: @team.leader">@team.name</a></li>
}
@for(report <- projects.ungroupedReportsInfo.toSeq.sortBy(p => p.projectName -> p.projectId -> p.subprojectNameOption)){
<li@if(report.subprojectNameOption.isEmpty){ class="base-project"}><a href="@link(Some("project:"+report.fullId))">@friendlyProjectName(report)</a></li>
}
</ul>
</div>
</div>
}
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li>@header.identity.fold{
<a class="btn btn-default" href="@routes.AuthController.signIn()">Log in</a>
}{ user =>
@form(routes.AuthController.signOut()){
@CSRF.formField
<button type="submit" class="btn btn-warning">Logout @user.username</button>
}
}</li>
</ul>
</div><!--/.nav-collapse -->
</div>
</nav>
<div class="container" id="main">
<h1>@title</h1>
@content
<hr>
That's all
</div>
</body>
</html>

View File

@@ -0,0 +1,5 @@
@(url: String)(content: Html)
@url match{
case NormalUrlPattern(_ @ _*) => {<a href="@url">@content</a>}
case "" => {@content}
}

View File

@@ -0,0 +1,4 @@
@(id: String, si: SnoozeInfo, collapseByDefault: Boolean)
<button style="float: right;" id="@id-snooze-button" type="button" class="btn btn-sm collapse @if(!si.shouldCollapse(collapseByDefault)){in}" data-toggle="collapse" data-target="#@id-snoozing" aria-label="Snooze">
<span aria-hidden="true">&#128564;</span>
</button>

View File

@@ -0,0 +1,9 @@
@(id: String, si: SnoozeInfo, versions: Map[String, Int])(implicit rh: RequestHeader, snoozes: SnoozesInfo, messages: Messages)
@import helper._
@form((routes.Application.snooze(id, versions): Call).withFragment(id), 'id -> s"$id-snoozing", 'class -> s"snoozing collapse${if(si.shouldExpandForm) "in" else ""}") {
@CSRF.formField
@inputText(si.form("until"), '_label -> "Snooze until", Symbol("data-provide") -> "datepicker", Symbol("data-date-format") -> "dd-mm-yyyy")
@inputText(si.form("reason"), '_label -> "Reason")
<button type="submit" class="btn btn-default">Snooze!</button>
}

View File

@@ -0,0 +1,16 @@
@(unused_id: String, si: SnoozeInfo, versions: Map[String, Int])(implicit requestHeader: RequestHeader)
@import helper._
@if(si.isSnoozed){
<h3>Snooze details</h3>
<ul class="snooze-list">
@for((snoozeId, snooze) <- si.snoozes){
<li>
@form(routes.Application.unsnooze(snoozeId, versions)) {
@snooze.reason until @snooze.until
@CSRF.formField
<button type="submit" class="btn btn-danger">×</button>
}
</li>
}
</ul>
}

View File

@@ -0,0 +1,18 @@
@(
projectsWithSelection: ProjectsWithSelection,
allDependencies: Seq[GroupedDependency]
)(implicit header: DefaultRequest)
@main(
title = s"All libraries for ${projectsWithSelection.projectNameText}",
projectsOption = Some((projectsWithSelection, routes.Statistics.allLibraries(_)))
){
@dependencyList(
"all",
allDependencies.sortBy(_.identifiers.toIndexedSeq.sortBy(i => (i.confidence.id, i.identifierType, i.name)).mkString(", ")),
selectorOption = projectsWithSelection.selectorString,
expandByDefault = false,
addButtons = false
)
}

View File

@@ -0,0 +1,187 @@
@(
projectsWithSelection: ProjectsWithSelection,
tagStatistics: Seq[Statistics.TagStatistics],
parsedReports: DependencyCheckReportsParser.Result
)(implicit messagesApi: MessagesApi, requestHeader: DefaultRequest)
@import com.ysoft.odc.CWE
@import controllers.Statistics.TagStatistics
@import play.api.libs.json.{JsNull, JsString}
@import scala.language.implicitConversions
@implicitTagStatistics(ts: TagStatistics) = @{ts.stats}
@he = {
<script type="text/javascript" src="@routes.Assets.versioned("lib/d3js/d3.js")"></script>
<script type="text/javascript" src="@routes.Assets.versioned("lib/jqplot/jquery.jqplot.min.js")"></script>
<script type="text/javascript" src="@routes.Assets.versioned("lib/jqplot/plugins/jqplot.barRenderer.min.js")"></script>
<script type="text/javascript" src="@routes.Assets.versioned("lib/jqplot/plugins/jqplot.categoryAxisRenderer.min.js")"></script>
<link type="text/css" rel="stylesheet" href="@routes.Assets.versioned("lib/jqplot/jquery.jqplot.min.css")">
<link type="text/css" rel="stylesheet" href="@routes.Assets.versioned("lib/tablesorter/css/theme.default.css")">
<script type="text/javascript" src="@routes.Assets.versioned("lib/tablesorter/js/jquery.tablesorter.min.js")"></script>
<script type="text/javascript" src="@routes.Assets.versioned("lib/StickyTableHeaders/js/jquery.stickytableheaders.min.js")"></script>
<script type="text/javascript">
$(function(){
$('.tablesorter').tablesorter();
$('.tablesorter').stickyTableHeaders({fixedOffset: $('#header')});
})
</script>
}
@plotData(frequency: Map[Option[CWE], Int]) = @{
import play.api.libs.json.Json._
val (ticks, details, values) = frequency.toSeq.sortBy{case (cweOption, _) => cweOption.map(c => c.numberOption -> c.name)}.map{
case (Some(cwe), freq) => (toJson(cwe.brief), toJson(cwe.name), freq)
case (None, freq) => (JsString("(none)"), JsNull, freq)
}.unzip3
toJson(Map(
"ticks" -> toJson(ticks),
"details" -> toJson(details),
"values" -> toJson(values)
))
}
@main(
title = s"statistics for ${projectsWithSelection.projectNameText}",
headExtension = he,
projectsOption = Some((projectsWithSelection, routes.Statistics.basic(_)))
){
All dependencies: @parsedReports.groupedDependencies.size <br>
Vulnerable dependencies: @parsedReports.vulnerableDependencies.size <br>
Vulnerabilities: @parsedReports.vulnerableDependencies.flatMap(_.vulnerabilities.map(_.name)).toSet.size<br>
Unique CPEs of vulnerable dependencies: @parsedReports.vulnerableDependencies.flatMap(_.cpeIdentifiers.map(_.toCpeIdentifierOption.get)).toSet.size <br>
Unique CPEs of all dependencies: @parsedReports.groupedDependencies.flatMap(_.cpeIdentifiers.map(_.toCpeIdentifierOption.get)).toSet.size <br>
@if(!projectsWithSelection.isProjectSpecified){
Multi-project dependencies: @parsedReports.groupedDependencies.filter(_.projects.size > 1).toSet.size <br>
}
<div id="weakness" data-data='@{plotData(Statistics.computeWeaknessesFrequency(parsedReports.groupedDependencies.flatMap(_.vulnerabilities).toSet))}'></div>
<script type="text/javascript">
var WeaknessIdentifier = function(brief, verbose){
this.brief = brief;
this.verbose = verbose;
};
WeaknessIdentifier.prototype.toString = function(){return "x"+this.brief;};
function initPlot(idPrefix, data){
var parentEl = $("#"+idPrefix);
var id = idPrefix+"-chart";
var detailsId = idPrefix+'-details';
if(parentEl.attr('data-initialized') == 'true'){
console.log('Not reinitializing the plot: ', idPrefix);
return;
}
console.log(parentEl);
parentEl.append($('<div>' ).attr({
"id": id,
"style" : "height: 300px; width: 900px;"
}));
parentEl.append($('<div>' ).attr({
"id": detailsId,
"style" : "height: 3ex; overflow: hidden;"
}));
var el = $("#"+id);
$.jqplot.config.enablePlugins = true;
data = data || JSON.parse(parentEl.attr('data-data'));
var s1 = data.values;
var ticks = data.ticks;
var details = data.details;
var plot1 = $.jqplot(id, [s1], {
animate: false, //!$.jqplot.use_excanvas, // Only animate if we're not using excanvas (not in IE 7 or IE 8)..
seriesDefaults:{
renderer:$.jqplot.BarRenderer,
pointLabels: { show: true }
},
axes: {
xaxis: {
renderer: $.jqplot.CategoryAxisRenderer,
ticks: ticks
}
},
highlighter: { show: false }
});
el.bind('jqplotDataClick',
function (ev, seriesIndex, pointIndex, data) {
//$('#info1').html('series: '+seriesIndex+', point: '+pointIndex+', data: '+data);
}
);
var detailsElement = $('#'+detailsId);
el.bind('jqplotDataUnhighlight', function(ev, seriesIndex, pointIndex, data){
detailsElement.text('');
});
el.bind('jqplotDataHighlight', function(ev, seriesIndex, pointIndex, data){
console.log('high', seriesIndex, pointIndex, data);
detailsElement.text((details[pointIndex]||"not described")+": "+s1[pointIndex]+"×");
});
el.attr('data-initialized', 'true');
};
$(document).ready(function(){
initPlot('weakness');
var n=0;
$('.stats').click(function(e){
console.log(e);
console.log(e.target);
var id = "modal-"+n;
n++;
var data = JSON.parse($(e.target).attr('data-data'));
var modalHeader = $('<div class="modal-header">' ).append($('<button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>' )).append($('<h4 class="modal-title">Modal title</h4>'));
var modalBody = $('<div class="modal-body"></div>').attr({id: id});
var modalFooter = $('<div class="modal-footer"></div>');
var modalDialog = $('<div class="modal-dialog">').append($('<div class="modal-content">').append(modalHeader).append(modalBody).append(modalFooter));
var modal = $('<div class="modal fade">').append(modalDialog );
modalDialog.css({
width: '930px',
//marginLeft: '-465px'
margin: 'auto'
});
modal.on('shown.bs.modal', function(){initPlot(id, data)});
modal.on('hidden.bs.modal', function(){modal.remove()});
$(document.body ).append(modal);
modal.modal({keyboard: true});
console.log(id, data);
return false;
});
});
</script>
<table class="table table-striped tablesorter">
<thead>
<tr>
<th>tag name</th>
<th># of vulns</th>
<th>vulnerable</th>
<th>all</th>
<th>vulnerable/all</th>
<th>CPE %</th>
<th>vulnerable/CPE</th>
</tr>
</thead>
<tbody>
@for(s <- tagStatistics){
<tr>
<td title="@s.tag.note">
<a href="@routes.Statistics.vulnerabilities(projectsWithSelection.selectorString, Some(s.tagId))" target="_blank" class="stats">@s.tag.name</a>
@*<div style="overflow: hidden; text-overflow: ellipsis; white-space: nowrap; max-width: 30em;">@s.tag.note</div>*@
</td>
<td class="text-right">
@s.vulnerabilities.size
<button type="button" class="glyphicon glyphicon-signal btn btn-xs stats" @if(s.vulnerabilities.isEmpty){disabled="disabled"} data-data="@plotData(s.weaknessesFrequency)"></button>
</td>
<td class="text-right">@s.vulnerableDependencies.size</td>
<td class="text-right">@s.dependencies.size</td>
<td class="text-right">@(f"${s.vulnerableRatio*100}%2.2f")&nbsp;%</td>
<td class="text-right">@(f"${s.cpeRatio*100}%2.2f")&nbsp;%</td>
<td class="text-right">@(f"${s.vulnerableDependencies.size.toDouble*100.0/s.dependenciesWithCpe.size.toDouble}%2.2f")&nbsp;%</td>
</tr>
}
</tbody>
</table>
}

View File

@@ -0,0 +1,40 @@
@(
projectsWithSelection: ProjectsWithSelection,
tagOption: Option[(Int, LibraryTag)],
statistics: Statistics.LibDepStatistics
)(implicit messagesApi: MessagesApi, requestHeader: DefaultRequest)
@main(
title = s"details for ${projectsWithSelection.projectNameText}${tagOption.map(_._2.name).fold("")(" and tag "+_)}",
projectsOption = Some((projectsWithSelection, x => routes.Statistics.vulnerabilities(x, tagOption.map(_._1))))
){
We have @statistics.vulnerabilitiesToDependencies.size vulnerabilities
of @statistics.vulnerabilitiesToDependencies.flatMap(_._2).toSet.size dependencies (@statistics.vulnerabilitiesToDependencies.flatMap(_._2.flatMap(_.plainLibraryIdentifiers)).toSet.size libaries).
@if(!projectsWithSelection.isProjectSpecified){
They are affecting @statistics.vulnerabilitiesToDependencies.flatMap(_._2.flatMap(_.projects)).toSet.size projects.
}else{
Just one project is selected.
<div class="alert alert-warning">When a project is selected, YSVSS might differ, as it is computed over a subset of subprojects. As a result, order of vulnerabilities might differ from their order at all-projects view.</div>
}
<div class="help">
Vulnerabilities are sorted by number of affected projects multiplied by their severity. If the score is the same, then they are sorted by severity. If even this matches, they are sorted by name (which is related to vulnerability age).
</div>
@for((vulnerability, dependencies) <- statistics.vulnerabilitiesToDependencies.toSeq.sortBy{case (vuln, deps) =>
(
vuln.ysvssScore(deps).map(-_), // total score
vuln.cvssScore.map(-_), // CVSS score
vuln.name // make it deterministic
)
}){
<h2><a href="@routes.Statistics.vulnerability(vulnerability.name, projectsWithSelection.selectorString)">@vulnerability.name</a>
<span class="severity">
(<span class="explained" title="vulnerability CVSS score">@(vulnerability.cvss.score.getOrElse{"?"})</span> ×
<span class="explained" title="number of affected projects">@dependencies.flatMap(_.projects).toSet.size</span> =
<span class="explained score" title="total score">@(vulnerability.ysvssScore(dependencies).fold{"?"}{d => f"$d%2.2f"})</span>
)</span>
</h2>
<p>@vulnerability.description</p>
@* <p>@dependencies.map(_.identifiers)</p> *@
@* <p>@dependencies.flatMap(_.projects).toSet</p> *@
}
}

View File

@@ -0,0 +1,78 @@
@(
vulnsAndVersionOption: Option[(Traversable[Vulnerability], String)],
cpes: Seq[String],
isDbOld: Boolean
)(implicit header: DefaultRequest)
@import helper._
@main(
title = "Vulnerabilities for a libary"
){
<script type="text/javascript">
function versionChanged(that){
function addClass(o, cl){o.addClass(cl)};
function removeClass(o, cl){o.removeClass(cl)};
var differentVersion = $(that).attr('data-version') != that.value;
$('.checked-version').css({color: differentVersion ? 'red' : ''});
var classForDifferentVersion = differentVersion ?addClass :removeClass;
var classForSameVersion = differentVersion ?removeClass :addClass;
classForDifferentVersion($('#submit-btn'), 'btn-primary');
classForSameVersion($('#different-version-warning'), 'hidden');
}
</script>
@form(routes.Statistics.searchVulnerableSoftware(Seq(), None), 'onsubmit->
"""
|return (function(f){
| var selectedCpes = $(f.elements.versionlessCpes).filter(function(i, x){return x.checked;}).map(function(i, x){return x.value;}).toArray()
| if(selectedCpes.length == 0){
| alert("Choose at least one CPE, please!");
| return false;
| }
|})(this);
|""".stripMargin
){
<label>
Version:
<input
type="text" name="versionOption" id="version-field" value="@vulnsAndVersionOption.fold("")(_._2)"
data-version="@vulnsAndVersionOption.fold("")(_._2)"
onkeypress="versionChanged(this)"
onkeyup="versionChanged(this)"
onchange="versionChanged(this)"
onpaste="versionChanged(this)"
oncut="versionChanged(this)"
>
@for((_, version) <- vulnsAndVersionOption){
<span id="different-version-warning" class="hidden">Note that you are viewing results for version <strong>@version</strong>!</span>
}
</label>
<ul>
@for(cpe <- cpes){
<li><label><input type="checkbox" name="versionlessCpes" value="@cpe" checked> @cpe</label></li>
}
</ul>
<button type="submit" class="btn btn-default" id="submit-btn">Check</button>
}
@if(isDbOld){
<div class="alert alert-warning">The vulnerability database seems to be outdated. Result might be thus inaccurate. Contact the administrator, please.</div>
}
@vulnsAndVersionOption.fold{
Select desired version, please
}{ case (vulns, version) =>
@if(vulns.isEmpty){
<div class="alert alert-success">No known vulnerabilities for version <strong class="checked-version">@version</strong>.</div>
}else{
<div class="alert alert-warning">There @if(vulns.size == 1){is one known vulnerability}else{are some known vulnerabilities} for version <strong class="checked-version">@version</strong>. Consider @if(vulns.size==1){its}else{their} impact before using the library, please.</div>
@for(vuln <- vulns.toIndexedSeq.sortBy(v => (v.cvssScore.map(-_), v.name))){
<h2>@vuln.name</h2>
@vulnerability("h3", s"vulnerability-${vuln.name}-details", vuln)
}
}
}
@*if(vulnsAndVersionOption.isEmpty){ *@
<script type="text/javascript">
document.getElementById("version-field").focus();
</script>
@* } *@
}

View File

@@ -0,0 +1,43 @@
@(
projectsWithSelection: ProjectsWithSelection,
vulnerability: Vulnerability,
affectedProjects: Map[ReportInfo, Set[GroupedDependency]],
vulnerableDependencies: Set[GroupedDependency],
affectedLibraries: Set[PlainLibraryIdentifier]
)(implicit header: DefaultRequest)
@section = @{views.html.genericSection("vuln")("h2") _}
@main(
title = s"vulnerability ${vulnerability.name} for ${projectsWithSelection.projectNameText}",
projectsOption = Some((projectsWithSelection, p => routes.Statistics.vulnerability(vulnerability.name, p)))
) {
@if(projectsWithSelection.isProjectSpecified){
<div class="alert alert-warning">The vulnerability details are limited to some subset of projects.<br><a class="btn btn-default" href="@routes.Statistics.vulnerability(vulnerability.name, None)">Show it for all projects!</a></div>
}
@section("details", "Vulnerability details") {
@views.html.vulnerability("h2", "vuln-details", vulnerability)
}
@section("affected-libs", s"Unique affected libraries  without version number (${affectedLibraries.size})"){
<ul>
@for(lib <- affectedLibraries){
<li>@lib</li>
}
</ul>
}
@section("affected-deps", s"Unique affected dependencies (${vulnerableDependencies.size})"){
<ul>
@for(dep <- vulnerableDependencies){
<li class="library-identification">@libraryIdentification(dep)</li>
}
</ul>
}
@section("affected-projects", s"Affected projects (${affectedProjects.size} projects with ${affectedProjects.flatMap(_._2).size} occurrences)"){
@for((project, dependencies) <- affectedProjects.toSeq.sortBy(_._1)){
<h3><a href="@routes.Statistics.basic(Some("project:"+project.fullId))">@friendlyProjectName(project)</a> (@dependencies.size)</h3>
<ul>
@for(dep <- dependencies.toSeq){
<li class="library-identification">@libraryIdentification(dep)</li>
}
</ul>
}
}
}

View File

@@ -0,0 +1,24 @@
@(
projectsWithSelection: ProjectsWithSelection,
name: String
)(implicit header: DefaultRequest)
@main(
title = s"Unknown vulnerability $name for ${projectsWithSelection.projectNameText}",
projectsOption = Some((projectsWithSelection, p => routes.Statistics.vulnerability(name, p)))
){
<div class="alert alert-warning">Vulnerability <i>@name</i> is not found@if(projectsWithSelection.isProjectSpecified){ for selected project(s)}.</div>
<h2>Possible solutions</h2>
<ul class="solutions">
@if(projectsWithSelection.isProjectSpecified){
<li>
Maybe the vulnerability does not affect this project, but it might affect other projects.<br>
<a class="btn btn-success" href="@routes.Statistics.vulnerability(name, None)">Look at all the projects!</a>
</li>
}
<li>
Maybe the vulnerability does not affect any of the projects.<br>
<a href="https://web.nvd.nist.gov/view/vuln/detail?vulnId=@helper.urlEncode(name)" class="btn btn-default">Look at NVD</a>
</li>
</ul>
}

View File

@@ -0,0 +1,64 @@
@(
projectsWithSelection: ProjectsWithSelection,
vulnerableDependencies: Seq[GroupedDependency],
allDependenciesCount: Int
)(implicit header: DefaultRequest)
@main(
title = s"Vulnerable libraries for ${projectsWithSelection.projectNameText} (${vulnerableDependencies.size} deps, ${vulnerableDependencies.flatMap(_.cpeIdentifiers.map(_.toCpeIdentifierOption.get)).toSet.size} CPEs)",
projectsOption = Some((projectsWithSelection, routes.Statistics.vulnerableLibraries(_)))
){
<script type="text/javascript" src="@routes.Assets.versioned("lib/jqplot/jquery.jqplot.min.js")"></script>
<script type="text/javascript" src="@routes.Assets.versioned("lib/jqplot/plugins/jqplot.pieRenderer.min.js")"></script>
<h2>Plot</h2>
<div id="vulnerable-dependencies-chart"></div>
<script type="text/javascript">
$(document).ready(function(){
var data = [
['Vulnerable', (@(vulnerableDependencies.size))], ['No known vulnerability', (@(allDependenciesCount - vulnerableDependencies.size))]
];
var plot1 = jQuery.jqplot ('vulnerable-dependencies-chart', [data], {
seriesDefaults: {
// Make this a pie chart.
renderer: jQuery.jqplot.PieRenderer,
rendererOptions: {
// Put data labels on the pie slices.
// By default, labels show the percentage of the slice.
showDataLabels: true,
dataLabels: 'value',
startAngle: -90,
seriesColors: ['red', 'green'],
legendOptions: {
textColor: 'white'
}
}
},
legend: { show:true, location: 'e' }
});
});
</script>
<h2>List</h2>
<div class="help">
<p>Libraries are sorted:</p>
<ol>
<li>by total score (max vulnerability score × number of affected dependencies) if vulnerability score is defined for at least one vulnerability</li>
<li>by affected dependency count if the score above is not defined</li>
<li>by number of vulnerabilities</li>
<li>by affected project count</li>
</ol>
<p>Note that the number of affected projects is calculated from the current view, not from all projects (unless all projects are selected).</p>
</div>
@dependencyList(
"vulnerable",
vulnerableDependencies.sortBy(d => (
d.ysdssScore.map(-_), // total score is the king
if(d.ysdssScore.isEmpty) Some(-d.dependencies.size) else None, // more affected dependencies if no vulnerability has defined severity
-d.vulnerabilities.size, // more vulnerabilities
-d.projects.size, // more affected projects
d.cpeIdentifiers.map(_.toCpeIdentifierOption.get).toSeq.sorted.mkString(" ")) // at least make the order deterministic
),
selectorOption = projectsWithSelection.selectorString,
expandByDefault = false,
addButtons = false
)
}

View File

@@ -0,0 +1,12 @@
@(dep: GroupedDependency, details: scala.xml.Node)
<pre>&lt;?xml version="1.0" encoding="UTF-8"?>
@((
<suppressions xmlns="https://www.owasp.org/index.php/OWASP_Dependency_Check_Suppression">
<suppress>
<notes>file name: {dep.fileNames.mkString(" OR ")}</notes>
<sha1>{dep.sha1}</sha1>
{details}
</suppress>
</suppressions>
).toString)
</pre>

View File

@@ -0,0 +1,9 @@
@(f: Form[String])(implicit requestHeader: DefaultRequest, messages: Messages)
@import helper._
@main("Data import"){
@form(action = controllers.routes.Application.tagsImportAction()){
@CSRF.formField
@textarea(f("data"))
<button type="submit">Import</button>
}
}

View File

@@ -0,0 +1,35 @@
@(ht: String, idPrefix: String, vuln: Vulnerability)
@row[T](name: String, render: T => String = {(_:T).toString})(valueOption: Option[T]) = {
@for(value <- valueOption){
<tr>
<th>@name</th>
<td>@render(value)</td>
</tr>
}
}
@section = @{views.html.genericSection(idPrefix)(ht) _}
<table class="vuln-details">
@row("CWE")(vuln.cweOption)
@row("CVSS: score")(vuln.cvss.score)
@row("CVSS: authenticationr")(vuln.cvss.authenticationr)
@row("CVSS: availability impact")(vuln.cvss.availabilityImpact)
@row("CVSS: access vector")(vuln.cvss.accessVector)
@row("CVSS: integrity impact")(vuln.cvss.integrityImpact)
@row("CVSS: access complexity")(vuln.cvss.accessComplexity)
@row("CVSS: confidential impact")(vuln.cvss.confidentialImpact)
</table>
@vuln.description
@section("vuln-sw", "Vulnerable software"){
<ul id="@idPrefix-details">
@for(sw <- vuln.vulnerableSoftware){
<li>@sw.name@if(sw.allPreviousVersion){ and all previous versions}</li>
}
</ul>
}
@section("references", "References"){
<ul>
@for(reference <- vuln.references){
<li>@secureLink(reference.url){@reference.source: @reference.name}</li>
}
</ul>
}

View File

@@ -0,0 +1,7 @@
@(name: String, count: Int, items: Traversable[_])
@name (@count)<br>
<ul>
@for(i <- items){
<li>@i</li>
}
</ul>

View File

@@ -0,0 +1,7 @@
@(name: String, items: Seq[(ReportInfo, Html)])
<strong>@name:</strong>
<ul>
@for((name, i) <- items){
<li>@friendlyProjectName(name): @i</li>
}
</ul>

View File

@@ -0,0 +1,5 @@
@(emptyReports: Seq[Build], urlBase: String)
<strong>Following projects have produced no results:</strong>
@for(build <- emptyReports.toSeq.sortBy(_.projectName)){
<li>@secureLink(build.resultLink(urlBase)){@build.projectName}</li>
}

View File

@@ -0,0 +1,7 @@
@(failedReports: Set[Build], urlBase: String)
<strong>There are some reports that failed to build:</strong>
<ul>
@for(build <- failedReports.toSeq.sortBy(_.projectName)){
<li>@secureLink(build.resultLink(urlBase)){@build.projectName}</li>
}
</ul>

View File

@@ -0,0 +1,19 @@
@(errors: Map[String, Throwable])
<ul>
Some reports failed to be downloaded:
@for((project, e) <- errors){
<li>
@project: @e
@(e match {
case upickle.Invalid.Data(data, msg) => s"$msg (data: $data)"
case upickle.Invalid.Json(msg, input) => s"$msg (input: $input)"
case _ => ""
})
</li>
@{
play.api.Logger.logger.error(s"Project results of $project failed to parse.", e)
()
}
}
</ul>

View File

@@ -0,0 +1,10 @@
@(items: IndexedSeq[GroupedDependency])(implicit rh: DefaultRequest, snoozes: SnoozesInfo, messages: Messages)
(ignore this item)
@groupedDependencyList(name = "", id = s"grouped-dependencies-warning-${java.util.UUID.randomUUID.toString}", collapse = false, allowSnoozes = false, versions = Map())(list = items)
@for(groupedDependency <- items){
<li>
<strong>@groupedDependency.fileNames.toSeq.sorted</strong>
@identifiers(groupedDependency.identifiers)
@groupedDependency.dependencies.keySet.groupBy(_.evidenceCollected)
</li>
}

View File

@@ -0,0 +1,5 @@
@(reportsWithErrorMessages: Seq[Build], urlBase: String)
<strong>Following projects seem to contain some error messages in their logs:</strong>
@for(build <- reportsWithErrorMessages.toSeq.sortBy(_.projectName)){
<li>@secureLink(build.resultLink(urlBase)){@build.projectName}</li>
}

View File

@@ -0,0 +1,2 @@
@(text: String)
@text

View File

@@ -0,0 +1,7 @@
@(unknownIdentifierTypes: Set[String])
There are some unknown identifier types. These will not be handled by the application:
<ul>
@for(identifierType <- unknownIdentifierTypes.toSeq.sorted){
<li>@identifierType</li>
}
</ul>

9
build.gradle Normal file
View File

@@ -0,0 +1,9 @@
task buildPlayApp(type:Exec){
//environment("JAVA_HOME", System.getProperty("java.home"))
if(System.getProperty("os.name").toLowerCase().startsWith("win")){
// unsupported OS because of silent fails; commandLine 'cmd', '/c', 'activator', 'stage'
commandLine 'report_error'
}else{
commandLine './activator', 'clean', 'dist'
}
}

97
build.sbt Normal file
View File

@@ -0,0 +1,97 @@
name := """odc-analyzer"""
version := "1.0-SNAPSHOT"
lazy val root = (project in file(".")).enablePlugins(PlayScala)
scalaVersion := "2.11.7"
resolvers += "Atlassian Releases" at "https://maven.atlassian.com/public/"
libraryDependencies ++= Seq(
//jdbc,
cache,
ws,
filters
//specs2 % Test
)
//resolvers += "scalaz-bintray" at https?"http://dl.bintray.com/scalaz/releases"
libraryDependencies += "com.lihaoyi" %% "upickle" % "0.3.4"
//libraryDependencies += "com.typesafe.play" %% "play-ws" % "2.4.2"
libraryDependencies += "com.jsuereth" %% "scala-arm" % "1.4"
libraryDependencies += "org.ccil.cowan.tagsoup" % "tagsoup" % "1.2.1"
libraryDependencies += "com.typesafe.play" %% "play-slick" % "1.1.1"
libraryDependencies += "com.typesafe.play" %% "play-slick-evolutions" % "1.1.1"
libraryDependencies += "com.github.tototoshi" %% "slick-joda-mapper" % "2.0.0"
//libraryDependencies += "nu.validator.htmlparser" % "htmlparser" % "1.2.1"
//libraryDependencies += "com.lihaoyi" %% "pprint" % "0.3.4"
libraryDependencies += "com.github.nscala-time" %% "nscala-time" % "2.0.0"
// libraryDependencies += "org.mariadb.jdbc" % "mariadb-java-client" % "1.1.9"
libraryDependencies += "org.postgresql" % "postgresql" % "9.4-1201-jdbc41"
libraryDependencies += "org.mariadb.jdbc" % "mariadb-java-client" % "1.3.3"
libraryDependencies += "org.webjars" % "bootstrap" % "3.3.5"
libraryDependencies += "org.webjars" % "jquery" % "2.1.4"
libraryDependencies += "org.webjars" % "bootstrap-datepicker" % "1.4.0"
libraryDependencies += "org.webjars" % "tablesorter" % "2.17.8"
libraryDependencies += "org.webjars.bower" % "StickyTableHeaders" % "0.1.17"
//libraryDependencies += "org.webjars.bower" % "plottable" % "1.5.0"
//libraryDependencies += "org.webjars" % "d3js" % "3.5.6"
libraryDependencies += "org.webjars" % "jqplot" % "1.0.8r1250"
//libraryDependencies += "com.github.mumoshu" %% "play2-memcached-play24" % "0.7.0"
libraryDependencies ++= Seq(
"com.mohiva" %% "play-silhouette" % "3.0.4",
"com.mohiva" %% "play-silhouette-testkit" % "3.0.4" % "test"
)
libraryDependencies += "net.codingwell" %% "scala-guice" % "4.0.0"
libraryDependencies += "net.ceedubs" %% "ficus" % "1.1.2"
libraryDependencies += "org.owasp" % "dependency-check-core" % "1.3.0"
routesImport += "binders.QueryBinders._"
// Uncomment to use Akka
//libraryDependencies += "com.typesafe.akka" %% "akka-actor" % "2.3.11"
// Play provides two styles of routers, one expects its actions to be injected, the
// other, legacy style, accesses its actions statically.
routesGenerator := InjectedRoutesGenerator
scalacOptions ++= Seq(
"-deprecation", // Emit warning and location for usages of deprecated APIs.
"-feature", // Emit warning and location for usages of features that should be imported explicitly.
"-unchecked", // Enable additional warnings where generated code depends on assumptions.
//"-Xfatal-warnings", // Fail the compilation if there are any warnings.
"-Xlint", // Enable recommended additional warnings.
"-Ywarn-adapted-args", // Warn if an argument list is modified to match the receiver.
"-Ywarn-dead-code", // Warn when dead code is identified.
"-Ywarn-inaccessible", // Warn about inaccessible types in method signatures.
"-Ywarn-nullary-override", // Warn when non-nullary overrides nullary, e.g. def foo() over def foo.
"-Ywarn-numeric-widen" // Warn when numerics are widened.
)

View File

@@ -0,0 +1,117 @@
# This is the main configuration file for the application.
# ~~~~~
# Secret key
# ~~~~~
# The secret key is used to secure cryptographics functions.
#
# This must be changed for production, but we recommend not changing it in this file.
#
# See https://www.playframework.com/documentation/latest/ApplicationSecret for more details.
play.crypto.secret = "changeme"
# The application languages
# ~~~~~
play.i18n.langs = [ "en" ]
play.modules.enabled += "modules.ConfigModule"
play.modules.enabled += "modules.SilhouetteModule"
yssdc{
bamboo{
url = …
}
reports {
provider = "bamboo"
bamboo{
user = …
password = …
}
}
projects = {jobId:humanReadableName, …}
teams = […]
exclusions{
missingGAV{
bySha1 = []
}
}
projectsToTeams = {
}
teamLeaders = { # all teams used here must be listed above
team: leader,
}
}
# Router
# ~~~~~
# Define the Router object to use for this application.
# This router will be looked up first when the application is starting up,
# so make sure this is the entry point.
# Furthermore, it's assumed your route file is named properly.
# So for an application router like `my.application.Router`,
# you may need to define a router file `conf/my.application.routes`.
# Default to Routes in the root package (and conf/routes)
# play.http.router = my.application.Routes
# Database configuration
# ~~~~~
# You can declare as many datasources as you want.
# By convention, the default datasource is named `default`
#
slick.dbs.default {
# Connection to internal database. It must be PostgreSQL.
driver = "slick.driver.PostgresDriver$"
db{
url = "jdbc:postgresql://localhost/odca"
user = …
password = …
}
}
slick.dbs.odc {
# Connection to ODC database. It should be MySQL/MariaDB. H2 DB is not supported. PostgreSQL might work if you get ODC working with it, Other databases might be supported in future.
driver = "slick.driver.MySQLDriver$"
db {
url = "jdbc:mysql://127.0.0.1/dependencycheck"
# Those credentials are default in ODC (but you might have changed them):
user = "dcuser"
password = "DC-Pass1337!"
}
}
# Evolutions
# ~~~~~
# You can disable evolutions if needed
# play.evolutions.enabled=false
# You can disable evolutions for a specific datasource if necessary
# play.evolutions.db.default.enabled=false
# If you want a persistent cache for development (it should speed up reload cycles), you might want to uncomment and adjust the following lines:
#play.modules.disabled+="play.api.cache.EhCacheModule"
#play.cache.path = "/home/user/.cache/odc-analysis"
silhouette {
# Authenticator settings
authenticator.cookieName = "authenticator"
authenticator.cookiePath = "/"
authenticator.secureCookie=false # is ignored; overriden in app/controllers/AuthController.scala; But it must be present!
authenticator.httpOnlyCookie = true
authenticator.useFingerprinting = true
authenticator.authenticatorIdleTimeout = 12 hours
authenticator.authenticatorExpiry = 12 hours
authenticator.rememberMe.cookieMaxAge = 30 days
authenticator.rememberMe.authenticatorIdleTimeout = 5 days
authenticator.rememberMe.authenticatorExpiry = 30 days
credentialsVerificationService{
type="allow-all" # accepts any credentials; allowed in dev mode only
#type="external" # verifies credentials at the URL specified below
#url="http://localhost:9050/"
}
}

View File

@@ -0,0 +1,34 @@
# --- !Ups
CREATE TABLE library (
id SERIAL,
library_type VARCHAR(255) NOT NULL, -- We could use enums, but it is too much bothering in PostgreSQL. We'll enforce those constrainst on application level :)
identifier VARCHAR(255) NOT NULL,
classified BOOLEAN,
PRIMARY KEY (id)
);
CREATE UNIQUE INDEX library_unique ON library (library_type, identifier);
CREATE TABLE library_tag (
id SERIAL,
name varchar(255) NOT NULL,
PRIMARY KEY (id)
);
CREATE UNIQUE INDEX library_tag_unique ON library_tag (name);
CREATE TABLE library_to_library_tag (
library_id INTEGER NOT NULL REFERENCES library,
library_tag_id INTEGER NOT NULL REFERENCES library_tag,
context_dependent BOOLEAN
);
CREATE UNIQUE INDEX library_to_library_tag_unique ON library_to_library_tag (library_id, library_tag_id);
# --- !Downs
DROP TABLE library;
DROP TABLE library_to_library_tag;
DROP TABLE library_tag;

View File

@@ -0,0 +1,7 @@
# --- !Ups
ALTER TABLE library_tag ADD COLUMN note VARCHAR(1024) NULL DEFAULT NULL;
# --- !Downs
ALTER TABLE library_tag DROP COLUMN note;

View File

@@ -0,0 +1,7 @@
# --- !Ups
ALTER TABLE library_tag ADD COLUMN warning_order INT NULL DEFAULT NULL;
# --- !Downs
ALTER TABLE library_tag DROP COLUMN warning_order;

View File

@@ -0,0 +1,13 @@
# --- !Ups
CREATE TABLE snooze(
"id" SERIAL NOT NULL,
"until" DATE NOT NULL,
"snoozed_object_identifier" VARCHAR(512) NOT NULL,
"reason" VARCHAR(1024) NOT NULL
);
CREATE INDEX snooze_until ON snooze (until);
# --- !Downs
DROP TABLE snooze;

View File

@@ -0,0 +1,18 @@
# --- !Ups
CREATE TABLE "cookie_authenticators" (
"id" VARCHAR NOT NULL,
"provider_id" VARCHAR NOT NULL,
"provider_key" VARCHAR NOT NULL,
"last_used" TIMESTAMP NOT NULL,
"expiration" TIMESTAMP NOT NULL,
"idle_timeout" BIGINT NULL,
"cookie_max_age" BIGINT NULL,
"fingerprint" VARCHAR NULL
);
CREATE INDEX cookie_authenticators_id ON cookie_authenticators (id);
# --- !Downs
DROP TABLE cookie_authenticators;

22
conf/logback.xml Normal file
View File

@@ -0,0 +1,22 @@
<configuration>
<conversionRule conversionWord="coloredLevel" converterClass="play.api.Logger$ColoredLevel" />
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%coloredLevel - %logger - %message%n%xException</pattern>
</encoder>
</appender>
<!--
The logger name is typically the Java/Scala package name.
This configures the log level to log at for a package and its children packages.
-->
<logger name="play" level="INFO" />
<logger name="slick.jdbc.JdbcBackend.statement" level="DEBUG" />
<logger name="application" level="DEBUG" />
<root level="WARN">
<appender-ref ref="STDOUT" />
</root>
</configuration>

Some files were not shown because too many files have changed in this diff Show More