commit 4b87ced31f0c4b5441c80ffcb6f1541ce459768f Author: Šesták Vít Date: Sun Jan 10 17:31:07 2016 +0100 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..96f1ebc --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +/RUNNING_PID +/logs/ +/project/*-shim.sbt +/project/project/ +/project/target/ +/target/ +.idea +conf/application.conf diff --git a/README.md b/README.md new file mode 100644 index 0000000..1e3bfd0 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +## Database + +I decided to use PostgreSQL, because + +* It is easy to set up +* It is reasonably strict by default. For example, when you try to insert a 256-character string in varchar(64), you get an error. (In MySQL, it gets silently truncated by default!) + + +## TODO + +### Naming +* Library × Identifier × PlainLibraryIdentifier – should be renamed + * Identifier is the most verbose one, it comes from OWASP Dependency Check. + * Library is a record stored in our database. + * PlainLibraryIdentifier is just version-less (e.g. `"$groupId:$artifactId"`) library identifier. diff --git a/activator b/activator new file mode 100755 index 0000000..c4e5755 --- /dev/null +++ b/activator @@ -0,0 +1,334 @@ +#!/usr/bin/env bash + +### ------------------------------- ### +### Helper methods for BASH scripts ### +### ------------------------------- ### + +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE=$(basename "$TARGET_FILE") + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE=$(readlink "$TARGET_FILE") + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE=$(basename "$TARGET_FILE") + COUNT=$(($COUNT + 1)) + done + + if [ "$TARGET_FILE" == "." -o "$TARGET_FILE" == ".." ]; then + cd "$TARGET_FILE" + TARGET_FILEPATH= + else + TARGET_FILEPATH=/$TARGET_FILE + fi + + # make sure we grab the actual windows path, instead of cygwin's path. + if ! is_cygwin; then + echo "$(pwd -P)/$TARGET_FILE" + else + echo $(cygwinpath "$(pwd -P)/$TARGET_FILE") + fi +) +} + +# TODO - Do we need to detect msys? + +# Uses uname to detect if we're in the odd cygwin environment. +is_cygwin() { + local os=$(uname -s) + case "$os" in + CYGWIN*) return 0 ;; + *) return 1 ;; + esac +} + +# This can fix cygwin style /cygdrive paths so we get the +# windows style paths. +cygwinpath() { + local file="$1" + if is_cygwin; then + echo $(cygpath -w $file) + else + echo $file + fi +} + +# Make something URI friendly +make_url() { + url="$1" + local nospaces=${url// /%20} + if is_cygwin; then + echo "/${nospaces//\\//}" + else + echo "$nospaces" + fi +} + +# Detect if we should use JAVA_HOME or just try PATH. +get_java_cmd() { + if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then + echo "$JAVA_HOME/bin/java" + else + echo "java" + fi +} + +echoerr () { + echo 1>&2 "$@" +} +vlog () { + [[ $verbose || $debug ]] && echoerr "$@" +} +dlog () { + [[ $debug ]] && echoerr "$@" +} +execRunner () { + # print the arguments one to a line, quoting any containing spaces + [[ $verbose || $debug ]] && echo "# Executing command line:" && { + for arg; do + if printf "%s\n" "$arg" | grep -q ' '; then + printf "\"%s\"\n" "$arg" + else + printf "%s\n" "$arg" + fi + done + echo "" + } + + exec "$@" +} +addJava () { + dlog "[addJava] arg = '$1'" + java_args=( "${java_args[@]}" "$1" ) +} +addApp () { + dlog "[addApp] arg = '$1'" + sbt_commands=( "${app_commands[@]}" "$1" ) +} +addResidual () { + dlog "[residual] arg = '$1'" + residual_args=( "${residual_args[@]}" "$1" ) +} +addDebugger () { + addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1" +} +addConfigOpts () { + dlog "[addConfigOpts] arg = '$*'" + for item in $* + do + addJava "$item" + done +} +# a ham-fisted attempt to move some memory settings in concert +# so they need not be messed around with individually. +get_mem_opts () { + local mem=${1:-1024} + local meta=$(( $mem / 4 )) + (( $meta > 256 )) || meta=256 + (( $meta < 1024 )) || meta=1024 + + # default is to set memory options but this can be overridden by code section below + memopts="-Xms${mem}m -Xmx${mem}m" + if [[ "${java_version}" > "1.8" ]]; then + extmemopts="-XX:MetaspaceSize=64m -XX:MaxMetaspaceSize=${meta}m" + else + extmemopts="-XX:PermSize=64m -XX:MaxPermSize=${meta}m" + fi + + if [[ "${java_opts}" == *-Xmx* ]] || [[ "${java_opts}" == *-Xms* ]] || [[ "${java_opts}" == *-XX:MaxPermSize* ]] || [[ "${java_opts}" == *-XX:ReservedCodeCacheSize* ]] || [[ "${java_opts}" == *-XX:MaxMetaspaceSize* ]]; then + # if we detect any of these settings in ${java_opts} we need to NOT output our settings. + # The reason is the Xms/Xmx, if they don't line up, cause errors. + memopts="" + extmemopts="" + fi + + echo "${memopts} ${extmemopts}" +} +require_arg () { + local type="$1" + local opt="$2" + local arg="$3" + if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then + die "$opt requires <$type> argument" + fi +} +is_function_defined() { + declare -f "$1" > /dev/null +} + +# If we're *not* running in a terminal, and we don't have any arguments, then we need to add the 'ui' parameter +detect_terminal_for_ui() { + [[ ! -t 0 ]] && [[ "${#residual_args}" == "0" ]] && { + addResidual "ui" + } + # SPECIAL TEST FOR MAC + [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]] && [[ "${#residual_args}" == "0" ]] && { + echo "Detected MAC OSX launched script...." + echo "Swapping to UI" + addResidual "ui" + } +} + +# Processes incoming arguments and places them in appropriate global variables. called by the run method. +process_args () { + while [[ $# -gt 0 ]]; do + case "$1" in + -h|-help) usage; exit 1 ;; + -v|-verbose) verbose=1 && shift ;; + -d|-debug) debug=1 && shift ;; + -mem) require_arg integer "$1" "$2" && app_mem="$2" && shift 2 ;; + -jvm-debug) + if echo "$2" | grep -E ^[0-9]+$ > /dev/null; then + addDebugger "$2" && shift + else + addDebugger 9999 + fi + shift ;; + -java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;; + -D*) addJava "$1" && shift ;; + -J*) addJava "${1:2}" && shift ;; + *) addResidual "$1" && shift ;; + esac + done + + is_function_defined process_my_args && { + myargs=("${residual_args[@]}") + residual_args=() + process_my_args "${myargs[@]}" + } +} + +# Actually runs the script. +run() { + # TODO - check for sane environment + + # process the combined args, then reset "$@" to the residuals + process_args "$@" + detect_terminal_for_ui + set -- "${residual_args[@]}" + argumentCount=$# + + #check for jline terminal fixes on cygwin + if is_cygwin; then + stty -icanon min 1 -echo > /dev/null 2>&1 + addJava "-Djline.terminal=jline.UnixTerminal" + addJava "-Dsbt.cygwin=true" + fi + + # run sbt + execRunner "$java_cmd" \ + "-Dactivator.home=$(make_url "$activator_home")" \ + $(get_mem_opts $app_mem) \ + ${java_opts[@]} \ + ${java_args[@]} \ + -jar "$app_launcher" \ + "${app_commands[@]}" \ + "${residual_args[@]}" + + local exit_code=$? + if is_cygwin; then + stty icanon echo > /dev/null 2>&1 + fi + exit $exit_code +} + +# Loads a configuration file full of default command line options for this script. +loadConfigFile() { + cat "$1" | sed '/^\#/d' +} + +### ------------------------------- ### +### Start of customized settings ### +### ------------------------------- ### +usage() { + cat < [options] + + Command: + ui Start the Activator UI + new [name] [template-id] Create a new project with [name] using template [template-id] + list-templates Print all available template names + -h | -help Print this message + + Options: + -v | -verbose Make this runner chattier + -d | -debug Set sbt log level to debug + -mem Set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem)) + -jvm-debug Turn on JVM debugging, open at the given port. + + # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) + -java-home Alternate JAVA_HOME + + # jvm options and output control + -Dkey=val Pass -Dkey=val directly to the java runtime + -J-X Pass option -X directly to the java runtime + (-J is stripped) + + # environment variables (read from context) + JAVA_OPTS Environment variable, if unset uses "" + SBT_OPTS Environment variable, if unset uses "" + ACTIVATOR_OPTS Environment variable, if unset uses "" + +In the case of duplicated or conflicting options, the order above +shows precedence: environment variables lowest, command line options highest. +EOM +} + +### ------------------------------- ### +### Main script ### +### ------------------------------- ### + +declare -a residual_args +declare -a java_args +declare -a app_commands +declare -r real_script_path="$(realpath "$0")" +declare -r activator_home="$(realpath "$(dirname "$real_script_path")")" +declare -r app_version="1.3.3" + +declare -r app_launcher="${activator_home}/activator-launch-${app_version}.jar" +declare -r script_name=activator +java_cmd=$(get_java_cmd) +declare -r java_opts=( "${ACTIVATOR_OPTS[@]}" "${SBT_OPTS[@]}" "${JAVA_OPTS[@]}" "${java_opts[@]}" ) +userhome="$HOME" +if is_cygwin; then + # cygwin sets home to something f-d up, set to real windows homedir + userhome="$USERPROFILE" +fi +declare -r activator_user_home_dir="${userhome}/.activator" +declare -r java_opts_config_home="${activator_user_home_dir}/activatorconfig.txt" +declare -r java_opts_config_version="${activator_user_home_dir}/${app_version}/activatorconfig.txt" + +# Now check to see if it's a good enough version +declare -r java_version=$("$java_cmd" -version 2>&1 | awk -F '"' '/version/ {print $2}') +if [[ "$java_version" == "" ]]; then + echo + echo No java installations was detected. + echo Please go to http://www.java.com/getjava/ and download + echo + exit 1 +elif [[ ! "$java_version" > "1.6" ]]; then + echo + echo The java installation you have is not up to date + echo Activator requires at least version 1.6+, you have + echo version $java_version + echo + echo Please go to http://www.java.com/getjava/ and download + echo a valid Java Runtime and install before running Activator. + echo + exit 1 +fi + +# if configuration files exist, prepend their contents to the java args so it can be processed by this runner +# a "versioned" config trumps one on the top level +if [[ -f "$java_opts_config_version" ]]; then + addConfigOpts $(loadConfigFile "$java_opts_config_version") +elif [[ -f "$java_opts_config_home" ]]; then + addConfigOpts $(loadConfigFile "$java_opts_config_home") +fi + +run "$@" diff --git a/activator-launch-1.3.3.jar b/activator-launch-1.3.3.jar new file mode 100644 index 0000000..59a2a46 Binary files /dev/null and b/activator-launch-1.3.3.jar differ diff --git a/activator.bat b/activator.bat new file mode 100644 index 0000000..266bb81 --- /dev/null +++ b/activator.bat @@ -0,0 +1,231 @@ +@REM activator launcher script +@REM +@REM Environment: +@REM In order for Activator to work you must have Java available on the classpath +@REM JAVA_HOME - location of a JDK home dir (optional if java on path) +@REM CFG_OPTS - JVM options (optional) +@REM Configuration: +@REM activatorconfig.txt found in the ACTIVATOR_HOME or ACTIVATOR_HOME/ACTIVATOR_VERSION +@setlocal enabledelayedexpansion + +@echo off + +set "var1=%~1" +if defined var1 ( + if "%var1%"=="help" ( + echo. + echo Usage activator [options] [command] + echo. + echo Commands: + echo ui Start the Activator UI + echo new [name] [template-id] Create a new project with [name] using template [template-id] + echo list-templates Print all available template names + echo help Print this message + echo. + echo Options: + echo -jvm-debug [port] Turn on JVM debugging, open at the given port. Defaults to 9999 if no port given. + echo. + echo Environment variables ^(read from context^): + echo JAVA_OPTS Environment variable, if unset uses "" + echo SBT_OPTS Environment variable, if unset uses "" + echo ACTIVATOR_OPTS Environment variable, if unset uses "" + echo. + echo Please note that in order for Activator to work you must have Java available on the classpath + echo. + goto :end + ) +) + +if "%ACTIVATOR_HOME%"=="" ( + set "ACTIVATOR_HOME=%~dp0" + @REM remove trailing "\" from path + set ACTIVATOR_HOME=!ACTIVATOR_HOME:~0,-1! +) + +set ERROR_CODE=0 +set APP_VERSION=1.3.3 +set ACTIVATOR_LAUNCH_JAR=activator-launch-%APP_VERSION%.jar + +rem Detect if we were double clicked, although theoretically A user could +rem manually run cmd /c +for %%x in (%cmdcmdline%) do if %%~x==/c set DOUBLECLICKED=1 + +rem FIRST we load a config file of extra options (if there is one) +set "CFG_FILE_HOME=%UserProfile%\.activator\activatorconfig.txt" +set "CFG_FILE_VERSION=%UserProfile%\.activator\%APP_VERSION%\activatorconfig.txt" +set CFG_OPTS= +if exist %CFG_FILE_VERSION% ( + FOR /F "tokens=* eol=# usebackq delims=" %%i IN ("%CFG_FILE_VERSION%") DO ( + set DO_NOT_REUSE_ME=%%i + rem ZOMG (Part #2) WE use !! here to delay the expansion of + rem CFG_OPTS, otherwise it remains "" for this loop. + set CFG_OPTS=!CFG_OPTS! !DO_NOT_REUSE_ME! + ) +) +if "%CFG_OPTS%"=="" ( + if exist %CFG_FILE_HOME% ( + FOR /F "tokens=* eol=# usebackq delims=" %%i IN ("%CFG_FILE_HOME%") DO ( + set DO_NOT_REUSE_ME=%%i + rem ZOMG (Part #2) WE use !! here to delay the expansion of + rem CFG_OPTS, otherwise it remains "" for this loop. + set CFG_OPTS=!CFG_OPTS! !DO_NOT_REUSE_ME! + ) + ) +) + +rem We use the value of the JAVACMD environment variable if defined +set _JAVACMD=%JAVACMD% + +if "%_JAVACMD%"=="" ( + if not "%JAVA_HOME%"=="" ( + if exist "%JAVA_HOME%\bin\java.exe" set "_JAVACMD=%JAVA_HOME%\bin\java.exe" + + rem if there is a java home set we make sure it is the first picked up when invoking 'java' + SET "PATH=%JAVA_HOME%\bin;%PATH%" + ) +) + +if "%_JAVACMD%"=="" set _JAVACMD=java + +rem Detect if this java is ok to use. +for /F %%j in ('"%_JAVACMD%" -version 2^>^&1') do ( + if %%~j==java set JAVAINSTALLED=1 + if %%~j==openjdk set JAVAINSTALLED=1 +) + +rem Detect the same thing about javac +if "%_JAVACCMD%"=="" ( + if not "%JAVA_HOME%"=="" ( + if exist "%JAVA_HOME%\bin\javac.exe" set "_JAVACCMD=%JAVA_HOME%\bin\javac.exe" + ) +) +if "%_JAVACCMD%"=="" set _JAVACCMD=javac +for /F %%j in ('"%_JAVACCMD%" -version 2^>^&1') do ( + if %%~j==javac set JAVACINSTALLED=1 +) + +rem BAT has no logical or, so we do it OLD SCHOOL! Oppan Redmond Style +set JAVAOK=true +if not defined JAVAINSTALLED set JAVAOK=false +if not defined JAVACINSTALLED set JAVAOK=false + +if "%JAVAOK%"=="false" ( + echo. + echo A Java JDK is not installed or can't be found. + if not "%JAVA_HOME%"=="" ( + echo JAVA_HOME = "%JAVA_HOME%" + ) + echo. + echo Please go to + echo http://www.oracle.com/technetwork/java/javase/downloads/index.html + echo and download a valid Java JDK and install before running Activator. + echo. + echo If you think this message is in error, please check + echo your environment variables to see if "java.exe" and "javac.exe" are + echo available via JAVA_HOME or PATH. + echo. + if defined DOUBLECLICKED pause + exit /B 1 +) + +rem Check what Java version is being used to determine what memory options to use +for /f "tokens=3" %%g in ('java -version 2^>^&1 ^| findstr /i "version"') do ( + set JAVA_VERSION=%%g +) + +rem Strips away the " characters +set JAVA_VERSION=%JAVA_VERSION:"=% + +rem TODO Check if there are existing mem settings in JAVA_OPTS/CFG_OPTS and use those instead of the below +for /f "delims=. tokens=1-3" %%v in ("%JAVA_VERSION%") do ( + set MAJOR=%%v + set MINOR=%%w + set BUILD=%%x + + set META_SIZE=-XX:MetaspaceSize=64M -XX:MaxMetaspaceSize=256M + if "!MINOR!" LSS "8" ( + set META_SIZE=-XX:PermSize=64M -XX:MaxPermSize=256M + ) + + set MEM_OPTS=!META_SIZE! + ) + +rem We use the value of the JAVA_OPTS environment variable if defined, rather than the config. +set _JAVA_OPTS=%JAVA_OPTS% +if "%_JAVA_OPTS%"=="" set _JAVA_OPTS=%CFG_OPTS% + +set DEBUG_OPTS= + +rem Loop through the arguments, building remaining args in args variable +set args= +:argsloop +if not "%~1"=="" ( + rem Checks if the argument contains "-D" and if true, adds argument 1 with 2 and puts an equal sign between them. + rem This is done since batch considers "=" to be a delimiter so we need to circumvent this behavior with a small hack. + set arg1=%~1 + if "!arg1:~0,2!"=="-D" ( + set "args=%args% "%~1"="%~2"" + shift + shift + goto argsloop + ) + + if "%~1"=="-jvm-debug" ( + if not "%~2"=="" ( + rem This piece of magic somehow checks that an argument is a number + for /F "delims=0123456789" %%i in ("%~2") do ( + set var="%%i" + ) + if defined var ( + rem Not a number, assume no argument given and default to 9999 + set JPDA_PORT=9999 + ) else ( + rem Port was given, shift arguments + set JPDA_PORT=%~2 + shift + ) + ) else ( + set JPDA_PORT=9999 + ) + shift + + set DEBUG_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=!JPDA_PORT! + goto argsloop + ) + rem else + set "args=%args% "%~1"" + shift + goto argsloop +) + +:run + +if "!args!"=="" ( + if defined DOUBLECLICKED ( + set CMDS="ui" + ) else set CMDS=!args! +) else set CMDS=!args! + +rem We add a / in front, so we get file:///C: instead of file://C: +rem Java considers the later a UNC path. +rem We also attempt a solid effort at making it URI friendly. +rem We don't even bother with UNC paths. +set JAVA_FRIENDLY_HOME_1=/!ACTIVATOR_HOME:\=/! +set JAVA_FRIENDLY_HOME=/!JAVA_FRIENDLY_HOME_1: =%%20! + +rem Checks if the command contains spaces to know if it should be wrapped in quotes or not +set NON_SPACED_CMD=%_JAVACMD: =% +if "%_JAVACMD%"=="%NON_SPACED_CMD%" %_JAVACMD% %DEBUG_OPTS% %MEM_OPTS% %ACTIVATOR_OPTS% %SBT_OPTS% %_JAVA_OPTS% "-Dactivator.home=%JAVA_FRIENDLY_HOME%" -jar "%ACTIVATOR_HOME%\%ACTIVATOR_LAUNCH_JAR%" %CMDS% +if NOT "%_JAVACMD%"=="%NON_SPACED_CMD%" "%_JAVACMD%" %DEBUG_OPTS% %MEM_OPTS% %ACTIVATOR_OPTS% %SBT_OPTS% %_JAVA_OPTS% "-Dactivator.home=%JAVA_FRIENDLY_HOME%" -jar "%ACTIVATOR_HOME%\%ACTIVATOR_LAUNCH_JAR%" %CMDS% + +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end + +@endlocal + +exit /B %ERROR_CODE% diff --git a/app/Filters.scala b/app/Filters.scala new file mode 100644 index 0000000..092927c --- /dev/null +++ b/app/Filters.scala @@ -0,0 +1,36 @@ +import javax.inject.Inject + +import play.api._ +import play.api.http.HttpFilters +import play.api.libs.iteratee.{Done, Iteratee} +import play.api.mvc._ +import play.filters.csrf.CSRFFilter +import play.twirl.api.Txt + +import scala.concurrent.Future + +class HostnameValidatingAction(allowedHostnames: Set[String], allowAllIps: Boolean, next: EssentialAction) extends EssentialAction with Results{ + + private val IpAddressPatternComponent = // comes from http://www.mkyong.com/regular-expressions/how-to-validate-ip-address-with-regular-expression/ + "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." + + "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." + + "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." + + "([01]?\\d\\d?|2[0-4]\\d|25[0-5])" + + private val IpAddress = ("""^"""+IpAddressPatternComponent+"""((:[0-9]+)?)$""").r + + override def apply(request: RequestHeader): Iteratee[Array[Byte], Result] = { + if( (allowedHostnames contains request.host) || (allowAllIps && IpAddress.findFirstMatchIn(request.host).isDefined )) next.apply(request) + else Iteratee.flatten(Future.successful(Done(Unauthorized(Txt(s"not allowed for host ${request.host}"))))) + } + +} + + +class HostnameFilter(allowedHostnames: Set[String], allowAllIps: Boolean = false) extends EssentialFilter { + override def apply(next: EssentialAction): EssentialAction = new HostnameValidatingAction(allowedHostnames, allowAllIps, next) +} + +class Filters @Inject() (csrfFilter: CSRFFilter, configuration: Configuration) extends HttpFilters { + def filters = Seq(csrfFilter, new HostnameFilter(configuration.getString("app.hostname").toSet, allowAllIps = true)) +} \ No newline at end of file diff --git a/app/assets/css/main.css b/app/assets/css/main.css new file mode 100644 index 0000000..68c9f35 --- /dev/null +++ b/app/assets/css/main.css @@ -0,0 +1,123 @@ +#main{ + padding-top: 50px; +} +.vuln-details{ + border-collapse: collapse; +} +.vuln-details td, .vuln-details th { + padding: 5px; + border: 1px solid gray; +} + +.toggle-warning { + float: left; + margin-right: 7px; +} + +.versionless-dependency:not(:hover) .related-links{ + display: none; +} + +.description{ + border-left: 3px solid black; + margin-bottom: 3px; + padding-left: 5px; +} +.badge a{ + color: yellow; +} +h2:before, h3:before, h4:before, h5:before, h6:before { + display: block; + content: " "; + margin-top: -50px; + height: 50px; + visibility: hidden; +} +#project-selector{ + float: left; + padding: 5px; + padding-top: 10px; +} +#project-selector button{ + max-width: 200px; + overflow: hidden; + text-overflow: ellipsis; +} +#project-selector .dropdown-menu{ + max-height: 300px; + overflow: auto; +} +#project-selector .base-project a{ + font-weight: bolder; +} + +h3.library-identification{ + border: 1px solid black; + padding: 5px; +} + +.library-identification-badge-hack{ + display: inline-block; + width: 1px; + padding-right: 0; + padding-left: 0; + margin-left: 0; + margin-right: 0; + visibility: hidden; +} + +.jqplot-table-legend{ + background-color: rgba(255, 255, 255, 0.25); + padding: 5px; + margin-right: 5px; +} + +.jqplot-table-legend-swatch{ + height: 10px; + width: 10px; + margin-right: 5px; +} +.jqplot-data-label{ + color: white; + font-weight: bold; +} + +.severity{ + font-size: smaller; + margin-bottom: 5px; +} + +.severity .score{ + font-weight: bolder; +} + +.explained{ + border-bottom: 1px dashed black; +} +.explained:after{ + content: "?"; + font-weight: bold; + display: inline-block; + color: white; + background-color: black; + border: 1px solid black; + border-radius: 100%; + text-align: center; + vertical-align: middle; + height: 1.5em; + width: 1.5em; + margin-left: 0.5em; + font-size: 50%; +} +.explained:hover{ + border-bottom-color: blue; +} +.explained:hover:after{ + background-color: blue; + border-color: blue; +} + +.help{ + border-left: 5px solid #00c7ff; + padding-left: 10px; +} \ No newline at end of file diff --git a/app/assets/js/main.js b/app/assets/js/main.js new file mode 100644 index 0000000..01f16e0 --- /dev/null +++ b/app/assets/js/main.js @@ -0,0 +1,59 @@ +function toggleTag(el){ + var btn = $(el); + var tagId = parseInt(btn.attr("data-tag-id")); + var libraryId = parseInt(btn.attr("data-library-id")); + btn.attr({disabled: true}); + var add = !btn.hasClass("btn-success"); + var libraryTagPair = {tagId: tagId, libraryId: libraryId}; + $.ajax({ + url: add ? Routes.addTag : Routes.removeTag, + method: 'POST', + //dataType: 'json', + data: JSON.stringify( + add ? { libraryTagPair: libraryTagPair, contextDependent: false} : libraryTagPair + ), + contentType : 'application/json', + success: function(){ + if(add){ + btn.addClass("btn-success"); + }else{ + btn.removeClass("btn-success"); + } + btn.attr({disabled: false}); + //alert("SUCCESS "+add); + }, + error: function(){ + alert("FAILED!"); + btn.addClass("btn-danger"); + // Can't enable the button as we can't be sure about the current state + }/*, + complete: function(a, b){ + console.log("complete", a, b); + alert(["complete", a, b]); + }*/ + }); +} +function toggleClassified(el){ + var btn = $(el); + var libraryId = parseInt(btn.attr("data-library-id")); + btn.attr({disabled: true}); + var classifiedNewValue = !btn.hasClass("btn-success"); + $.ajax({ + url: Routes.controllers.Application.setClassified(classifiedNewValue).url, + method: 'POST', + contentType : 'application/json', + data: ""+libraryId, + success: function(){ + if(classifiedNewValue){ + btn.addClass("btn-success"); + }else{ + btn.removeClass("btn-success"); + } + btn.attr({disabled: false}); + }, + error: function(){ + alert("FAILED!"); + btn.addClass("btn-danger"); + } + }); +} \ No newline at end of file diff --git a/app/binders/QueryBinders.scala b/app/binders/QueryBinders.scala new file mode 100644 index 0000000..da10186 --- /dev/null +++ b/app/binders/QueryBinders.scala @@ -0,0 +1,38 @@ +package binders + +import java.net.URLDecoder.decode +import java.net.URLEncoder.encode + +import play.api.mvc.{JavascriptLiteral, PathBindable, QueryStringBindable} + +object QueryBinders { + + /*private def bindableSet[T](implicit seqBinder: QueryStringBindable[Seq[T]]): QueryStringBindable[Set[T]] = seqBinder.transform( + _.toSet, + _.toSeq + ) + + implicit def bindableSetOfInt(implicit seqBinder: QueryStringBindable[Seq[Int]]): QueryStringBindable[Set[Int]] = bindableSet[Int]*/ + + import play.api.libs.json._ + private val formats = implicitly[Format[Map[String, Int]]] + + implicit def bindableMapStringToInt: QueryStringBindable[Map[String, Int]] = { + QueryStringBindable.bindableString.transform(s => formats.reads(Json.parse(s)).getOrElse(Map()), map => formats.writes(map).toString()) + } + + implicit object MapStringIntJavascriptLiteral extends JavascriptLiteral[Map[String, Int]] { + override def to(value: Map[String, Int]): String = formats.writes(value).toString() + } + + implicit val StringOptionPathBindable: PathBindable[Option[String]] = implicitly[PathBindable[String]].transform( + { + case "" => None + case x => Some(decode(x, "utf-8")) + }, + _.map(encode(_, "utf-8")).getOrElse("") + ) + + //implicit def somePathBindable[T : PathBindable]: PathBindable[Some[T]] = implicitly[PathBindable[T]].transform(Some(_), _.x) + +} diff --git a/app/com/ysoft/odc/BambooDownloader.scala b/app/com/ysoft/odc/BambooDownloader.scala new file mode 100644 index 0000000..b360c28 --- /dev/null +++ b/app/com/ysoft/odc/BambooDownloader.scala @@ -0,0 +1,166 @@ +package com.ysoft.odc + +import com.google.inject.Inject +import com.google.inject.name.Named +import org.ccil.cowan.tagsoup.jaxp.SAXFactoryImpl +import play.api.libs.ws.{WS, WSAuthScheme, WSClient, WSRequest} +import upickle.default._ + +import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Success, Try} +import scala.xml.Node + +final case class Link( + href: String, + rel: String +) + +final case class Artifact( + name: String, + link: Link + //size: Option[Long] +){ + def url: String = link.href +} + +final case class Artifacts( + size: Int, + //`start-index`: Int, + //`max-result`: Int + artifact: Seq[Artifact] +) + +final case class Build( + state: String, + //link: Link, + buildResultKey: String, + buildState: String, + projectName: String, + artifacts: Artifacts +) { + def resultLink(urlBase: String): String = s"$urlBase/browse/$buildResultKey/log" +} +sealed trait FlatArtifactItem{ + def name: String +} +abstract sealed class ArtifactItem{ + def name: String + final def flatFiles: Map[String, Array[Byte]] = flatFilesWithPrefix("") + def flatFilesWithPrefix(prefix: String): Map[String, Array[Byte]] + def toTree(indent: Int = 0): String + def toTree: String = toTree(0) +} +final case class ArtifactFile(name: String, data: Array[Byte]) extends ArtifactItem with FlatArtifactItem{ + override def toTree(indent: Int): String = " "*indent + s"$name = $data" + override def flatFilesWithPrefix(prefix: String): Map[String, Array[Byte]] = Map(prefix + name -> data) + def dataString = new String(data, "utf-8") +} +final case class ArtifactDirectory(name: String, items: Map[String, ArtifactItem]) extends ArtifactItem{ + override def toTree(indent: Int): String = " "*indent + s"$name:\n"+items.values.map(_.toTree(indent+2)).mkString("\n") + override def flatFilesWithPrefix(prefix: String): Map[String, Array[Byte]] = items.values.flatMap(_.flatFilesWithPrefix(s"$prefix$name/")).toMap +} +final case class FlatArtifactDirectory(name: String, items: Seq[(String, String)]) extends FlatArtifactItem{} + +trait BambooAuthentication{ + def addAuth(request: WSRequest): WSRequest +} + +class SessionIdBambooAuthentication(sessionId: String) extends BambooAuthentication{ + override def addAuth(request: WSRequest): WSRequest = request.withHeaders("Cookie" -> s"JSESSIONID=${sessionId.takeWhile(_.isLetterOrDigit)}") +} + +class CredentialsBambooAuthentication(user: String, password: String) extends BambooAuthentication{ + override def addAuth(request: WSRequest): WSRequest = request.withQueryString("os_authType" -> "basic").withAuth(user, password, WSAuthScheme.BASIC) +} + +final class BambooDownloader @Inject() (@Named("bamboo-server-url") val server: String, auth: BambooAuthentication)(implicit executionContext: ExecutionContext, wSClient: WSClient) extends Downloader { + + private object ArtifactKeys{ + val BuildLog = "Build log" + val ResultsHtml = "Report results-HTML" + val ResultsXml = "Report results-XML" + } + + private def downloadArtifact(artifactMap: Map[String, Artifact], key: String)(implicit wSClient: WSClient): Future[FlatArtifactItem] = { + val artifact = artifactMap(key) + downloadArtifact(artifact.url, artifact.name) + } + + private def downloadArtifact(url: String, name: String)(implicit wSClient: WSClient): Future[FlatArtifactItem] = { + bambooUrl(url).get().map{response => + response.header("Content-Disposition") match{ + case Some(_) => ArtifactFile(name = name, data = response.bodyAsBytes) + case None => + val html = response.body + val hpf = new SAXFactoryImpl + hpf.setFeature("http://xml.org/sax/features/external-general-entities", false) + //hpf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true) + hpf.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + val HtmlParser = hpf.newSAXParser() + val Html = scala.xml.XML.withSAXParser(HtmlParser) + val xml = Html.loadString(html) + val tds = xml \\ "td" + val subdirs = tds flatMap { td => + (td \ "img").headOption.flatMap{img => + val suffix = img.attribute("alt").map(_.text) match { // suffix seems to be no longer needed, as we recognize directories elsehow + case Some("(dir)") => "/" + case Some("(file)") => "" + case other => sys.error(s"unexpected directory item type: $other") + } + (td \ "a").headOption.map{ link => + val hrefAttribute: Option[Seq[Node]] = link.attribute("href") + link.text -> (hrefAttribute.getOrElse(sys.error(s"bad link $link at $url")).text+suffix) : (String, String) + } : Option[(String, String)] + } : Option[(String, String)] + } + FlatArtifactDirectory(name = name, items = subdirs) + } + } + } + + private def downloadArtifactRecursively(artifactMap: Map[String, Artifact], key: String)(implicit wSClient: WSClient): Future[ArtifactItem] = { + val artifact = artifactMap(key) + downloadArtifactRecursively(url = artifact.url, name = artifact.name) + } + + private def downloadArtifactRecursively(url: String, name: String/*artifactMap: Map[String, Artifact], key: String*/)(implicit wSClient: WSClient): Future[ArtifactItem] = { + downloadArtifact(url/*artifactMap, key*/, name).flatMap{ + case directoryStructure: FlatArtifactDirectory => + Future.traverse(directoryStructure.items){case (subName, urlString) => + downloadArtifactRecursively(server+urlString, subName) + }.map{ items => + ArtifactDirectory(name = directoryStructure.name, items = items.map(i => i.name->i).toMap) + } + case file: ArtifactFile => Future.successful(file) + } + } + + override def downloadProjectReports(projects: Set[String], requiredVersions: Map[String, Int]): Future[(Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable])] = { + val resultsFuture = Future.traverse(projects){project => + downloadProjectReport(project, requiredVersions.get(project)) + } + resultsFuture.map{ results => + val (successfulReportTries, failedReportTries) = results.partition(_._2.isSuccess) + val successfulReports = successfulReportTries.map{case (name, Success(data)) => name -> data; case _ => ???}.toMap + val failedReports = failedReportTries.map{case (name, Failure(data)) => name -> data; case _ => ???}.toMap + (successfulReports, failedReports) + } + } + + private def bambooUrl(url: String) = auth.addAuth(WS.clientUrl(url)) + + private def downloadProjectReport(project: String, versionOption: Option[Int]): Future[(String, Try[(Build, ArtifactItem, ArtifactFile)])] = { + val url = s"$server/rest/api/latest/result/$project-${versionOption.getOrElse("latest")}.json?expand=artifacts" + val resultFuture = (bambooUrl(url).get().flatMap { response => + val build = read[Build](response.body) + val artifactMap: Map[String, Artifact] = build.artifacts.artifact.map(x => x.name -> x).toMap + val logFuture = downloadArtifact(artifactMap, ArtifactKeys.BuildLog).map(_.asInstanceOf[ArtifactFile]) + val reportsFuture: Future[ArtifactItem] = downloadArtifactRecursively(artifactMap, ArtifactKeys.ResultsXml) + for { + log <- logFuture + reports <- reportsFuture + } yield (build, reports, log) + }: Future[(Build, ArtifactItem, ArtifactFile)]) + resultFuture.map(data => project -> Success(data)).recover{case e => project -> Failure(e)} + } +} diff --git a/app/com/ysoft/odc/Checks.scala b/app/com/ysoft/odc/Checks.scala new file mode 100644 index 0000000..bdff3f4 --- /dev/null +++ b/app/com/ysoft/odc/Checks.scala @@ -0,0 +1,33 @@ +package com.ysoft.odc + +import controllers.WarningSeverity.WarningSeverity +import controllers.{IdentifiedWarning, ReportInfo, Warning} +import play.twirl.api.{Html, HtmlFormat} + +object Checks { + + def differentValues(id: String, name: String, severity: WarningSeverity)(f: Map[ReportInfo, Analysis] => Traversable[_]) = { (data: Map[ReportInfo, Analysis]) => + val variants = f(data) + if(variants.size > 1){ + Some(IdentifiedWarning(id, HtmlFormat.escape(s"different $name!"), severity)) + }else{ + None + } + } + + def badValues(id: String, name: String, severity: WarningSeverity)(f: (ReportInfo, Analysis) => Option[Html]): Map[ReportInfo, Analysis] => Option[Warning] = { (data: Map[ReportInfo, Analysis]) => + val badValues = data.collect(Function.unlift{case (analysisName, analysis) => f(analysisName, analysis).map(analysisName -> _)}).toSeq + if(badValues.size > 0) Some(IdentifiedWarning(id, views.html.warnings.badValues(name, badValues), severity)) + else None + } + + def badGroupedDependencies[C <: Traversable[_]](id: String, name: String, severity: WarningSeverity)(f: Seq[GroupedDependency] => C)(show: C => Traversable[_] = {(x: C) => x}, exclusions: Set[Exclusion] = Set()): (Seq[GroupedDependency] => Option[Warning]) = { (data: Seq[GroupedDependency]) => + val badItems = f(data.filterNot(ds => exclusions.exists(_.matches(ds)))) + if(badItems.size > 0){ + Some(IdentifiedWarning(id, views.html.warnings.badGroupedDependencies(name, badItems.size, show(badItems)), severity)) + }else{ + None + } + } + +} diff --git a/app/com/ysoft/odc/Downloader.scala b/app/com/ysoft/odc/Downloader.scala new file mode 100644 index 0000000..c772b43 --- /dev/null +++ b/app/com/ysoft/odc/Downloader.scala @@ -0,0 +1,10 @@ +package com.ysoft.odc +import scala.concurrent.Future + +/** + * Created by user on 10/30/15. + */ +trait Downloader { + + def downloadProjectReports(projects: Set[String], requiredVersions: Map[String, Int]): Future[(Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable])] +} diff --git a/app/com/ysoft/odc/LocalFilesDownloader.scala b/app/com/ysoft/odc/LocalFilesDownloader.scala new file mode 100644 index 0000000..0a1f217 --- /dev/null +++ b/app/com/ysoft/odc/LocalFilesDownloader.scala @@ -0,0 +1,17 @@ +package com.ysoft.odc + +import javax.inject.Named + +import com.google.inject.Inject + +import scala.concurrent.Future + +class LocalFilesDownloader @Inject() (@Named("reports-path") path: String) extends Downloader{ + override def downloadProjectReports(projects: Set[String], requiredVersions: Map[String, Int]): Future[(Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable])] = { + if(requiredVersions != Map()){ + sys.error("Versions are not supported there") + } + projects.map{pn => ???} + ??? + } +} diff --git a/app/com/ysoft/odc/OdcParser.scala b/app/com/ysoft/odc/OdcParser.scala new file mode 100644 index 0000000..9f0f5f3 --- /dev/null +++ b/app/com/ysoft/odc/OdcParser.scala @@ -0,0 +1,293 @@ +package com.ysoft.odc + +import com.github.nscala_time.time.Imports._ +import controllers.ReportInfo +import models.{LibraryType, PlainLibraryIdentifier} + +import scala.xml._ + +final case class SerializableXml private (xmlString: String, @transient private val xmlData: NodeSeq) extends Serializable{ + @transient lazy val xml = Option(xmlData).getOrElse(SecureXml.loadString(xmlString)) + + override def equals(obj: scala.Any): Boolean = obj match { + case SerializableXml(s, _) => s == this.xmlString + } + + override def hashCode(): Int = 42+xmlString.hashCode + +} + +object SerializableXml{ + def apply(xml: Node): SerializableXml = SerializableXml(xml.toString(), xml) + def apply(xml: NodeSeq): SerializableXml = SerializableXml(xml.toString(), xml) +} + +final case class Analysis(scanInfo: SerializableXml, name: String, reportDate: DateTime, dependencies: Seq[Dependency]) + +final case class Hashes(sha1: String, md5: String){ + override def toString: String = s"Hashes(sha1=$sha1, md5=$md5)" +} + +final case class Exclusion(sha1: String) extends AnyVal { + def matches(dependency: Dependency): Boolean = dependency.sha1 == sha1 + def matches(group: GroupedDependency): Boolean = group.sha1 == sha1 +} + +final case class Evidence(source: String, name: String, value: String, confidence: String, evidenceType: String) + +final case class Dependency( + fileName: String, + filePath: String, + md5: String, + sha1: String, + description: String, + evidenceCollected: Set[Evidence], + identifiers: Seq[Identifier], + license: String, + vulnerabilities: Seq[Vulnerability], + suppressedVulnerabilities: Seq[Vulnerability], + relatedDependencies: SerializableXml +){ + def hashes = Hashes(sha1 = sha1, md5 = md5) + + def plainLibraryIdentifiers: Set[PlainLibraryIdentifier] = identifiers.flatMap(_.toLibraryIdentifierOption).toSet + + + /* + Method equals seems to be a CPU hog there. I am not sure if we can do something reasonable about it. + We can compare by this.hashes, but, in such case, dependencies that differ in evidence will be considered the same if their JAR hashes are the same, which would break some sanity checks. + */ +} + +/** + * A group of dependencies having the same fingerprints + * @param dependencies + */ +final case class GroupedDependency(dependencies: Map[Dependency, Set[ReportInfo]]) { + def parsedDescriptions: Seq[Seq[Seq[String]]] = descriptions.toSeq.sorted.map(_.split("\n\n").toSeq.map(_.split("\n").toSeq)) + def isVulnerable: Boolean = vulnerabilities.nonEmpty + def maxCvssScore = (Seq(None) ++ vulnerabilities.map(_.cvssScore)).max + def ysdssScore = maxCvssScore.map(_ * projects.size) + def descriptions = dependencies.keySet.map(_.description) + def projects = dependencies.values.flatten.toSet + def fileNames = dependencies.keySet.map(_.fileName) + def hashes = dependencies.keys.head.hashes // valid since all deps in a group have the same hashes + val sha1 = hashes.sha1 + def identifiers: Set[Identifier] = dependencies.keySet.flatMap(_.identifiers) + def mavenIdentifiers = identifiers.filter(_.identifierType == "maven") + def cpeIdentifiers = identifiers.filter(_.identifierType == "cpe") + def vulnerabilities: Set[Vulnerability] = dependencies.keySet.flatMap(_.vulnerabilities) + def plainLibraryIdentifiers: Set[PlainLibraryIdentifier] = identifiers.flatMap(_.toLibraryIdentifierOption) + def hasCpe: Boolean = cpeIdentifiers.nonEmpty +} + +object GroupedDependency{ + def apply(deps: Seq[(Dependency, ReportInfo)]): GroupedDependency = GroupedDependency(deps.groupBy(_._1).mapValues(_.map(_._2).toSet)) // TODO: the groupBy seems to be a CPU hog (because of GroupedDependency.equals); The mapValues is lazy, so its repeated might also be a performance hog, but I doubt that values are used frequently. +} + +object Confidence extends Enumeration { + type Confidence = Value + // Order is important + val Low = Value("LOW") + val Medium = Value("MEDIUM") + val High = Value("HIGH") + val Highest = Value("HIGHEST") + +} + +final case class Reference(source: String, url: String, name: String) + +final case class VulnerableSoftware(allPreviousVersion: Boolean, name: String) + +final case class CvssRating(score: Option[Double], authenticationr: Option[String], availabilityImpact: Option[String], accessVector: Option[String], integrityImpact: Option[String], accessComplexity: Option[String], confidentialImpact: Option[String]) + +final case class CWE(name: String) extends AnyVal{ + override def toString = name + def brief = name.takeWhile(_ != ' ') + def numberOption: Option[Int] = if(brief startsWith "CWE-") try { + Some(brief.substring(4).toInt) + } catch { + case _: NumberFormatException => None + } else None +} + +final case class Vulnerability(name: String, cweOption: Option[CWE], cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){ + def cvssScore = cvss.score + def ysvssScore(affectedDeps: Set[GroupedDependency]) = cvssScore.map(_ * affectedDeps.flatMap(_.projects).toSet.size) +} + +final case class Identifier(name: String, confidence: Confidence.Confidence, url: String, identifierType: String) { + def toLibraryIdentifierOption: Option[PlainLibraryIdentifier] = { + if(identifierType == "maven"){ + val groupId::artifactId::_ = name.split(':').toList + Some(PlainLibraryIdentifier(libraryType = LibraryType.Maven, libraryIdentifier = s"$groupId:$artifactId")) + }else{ + None + } + } + def toCpeIdentifierOption: Option[String] = identifierType match { + case "cpe" => Some(name) + case _ => None + } + //def isClassifiedInSet(set: Set[PlainLibraryIdentifier]): Boolean = toLibraryIdentifierOption.exists(set contains _) +} + +object OdcParser { + + def filterWhitespace(node: Node) = node.nonEmptyChildren.filter{ + case t: scala.xml.Text if t.text.trim == "" => false + case t: scala.xml.PCData if t.text.trim == "" => false + case _ => true + } + + def checkElements(node: Node, knownElements: Set[String]) { + val subelementNames = filterWhitespace(node).map(_.label).toSet + val unknownElements = subelementNames -- knownElements + if(unknownElements.nonEmpty){ + sys.error("Unknown elements for "+node.label+": "+unknownElements) + } + } + + private def getAttributes(data: MetaData): List[String] = data match { + case Null => Nil + case Attribute(key, _, next) => key :: getAttributes(next) + } + + def checkParams(node: Node, knownParams: Set[String]) { + val paramNames = getAttributes(node.attributes).toSet + val unknownParams = paramNames -- knownParams + if(unknownParams.nonEmpty){ + sys.error("Unknown params for "+node.label+": "+unknownParams) + } + } + + + def parseVulnerableSoftware(node: Node): VulnerableSoftware = { + checkElements(node, Set("#PCDATA")) + checkParams(node, Set("allPreviousVersion")) + if(node.label != "software"){ + sys.error(s"Unexpected element for vulnerableSoftware: ${node.label}") + } + VulnerableSoftware( + name = node.text, + allPreviousVersion = node.attribute("allPreviousVersion").map(_.text).map(Map("true"->true, "false"->false)).getOrElse(false) + ) + } + + def parseReference(node: Node): Reference = { + checkElements(node, Set("source", "url", "name")) + checkParams(node, Set()) + if(node.label != "reference"){ + sys.error(s"Unexpected element for reference: ${node.label}") + } + Reference( + source = (node \ "source").text, + url = (node \ "url").text, + name = (node \ "name").text + ) + } + + def parseVulnerability(node: Node, expectedLabel: String = "vulnerability"): Vulnerability = { + checkElements(node, Set("name", "severity", "cwe", "cvssScore", "description", "references", "vulnerableSoftware", "cvssAuthenticationr", "cvssAvailabilityImpact", "cvssAccessVector", "cvssIntegrityImpact", "cvssAccessComplexity", "cvssConfidentialImpact")) + if(node.label != expectedLabel){ + sys.error(s"Unexpected element for vuln: ${node.label}") + } + def t(ns: NodeSeq) = { + ns match { + case Seq() => None + case Seq(one) => + one.attributes match { + case Null => + one.child match { + case Seq(hopefullyTextChild) => + hopefullyTextChild match { + case Text(data) => Some(data) + } + } + } + } + } + Vulnerability( + name = (node \ "name").text, + //severity = (node \ "severity"), <- severity is useless, as it is computed from cvssScore :D + cweOption = (node \ "cwe").headOption.map(_.text).map(CWE), + description = (node \ "description").text, + cvss = CvssRating( + score = (node \ "cvssScore").headOption.map(_.text.toDouble), + authenticationr = t(node \ "cvssAuthenticationr"), + availabilityImpact = t(node \ "cvssAvailabilityImpact"), + accessVector = t(node \ "cvssAccessVector"), + integrityImpact = t(node \ "cvssIntegrityImpact"), + accessComplexity = t(node \ "cvssAccessComplexity"), + confidentialImpact = t(node \ "cvssConfidentialImpact") + ), + references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_)), + vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware) + ) + } + + def parseIdentifier(node: Node): Identifier = { + checkElements(node, Set("name", "url")) + checkParams(node, Set("type", "confidence")) + val ExtractPattern = """\((.*)\)""".r + Identifier( + name = (node \ "name").text match { + case ExtractPattern(text) => text + }, + url = (node \ "url").text, + identifierType = node.attribute("type").get.text, + confidence = Confidence.withName(node.attribute("confidence").get.text) + ) + } + + def parseIdentifiers(seq: Node): Seq[Identifier] = { + filterWhitespace(seq.head).map(parseIdentifier(_)) + } + + def parseDependency(node: Node): Dependency = { + checkElements(node, Set("fileName", "filePath", "md5", "sha1", "description", "evidenceCollected", "identifiers", "license", "vulnerabilities", "relatedDependencies")) + checkParams(node, Set()) + val (vulnerabilities: Seq[Node], suppressedVulnerabilities: Seq[Node]) = (node \ "vulnerabilities").headOption.map(filterWhitespace).getOrElse(Seq()).partition(_.label == "vulnerability") + Dependency( + fileName = (node \ "fileName").text, + filePath = (node \ "filePath").text, + md5 = (node \ "md5").text, + sha1 = (node \ "sha1").text, + description = (node \ "description").text, + evidenceCollected = filterWhitespace((node \ "evidenceCollected").head).map(parseEvidence).toSet, + identifiers = (node \ "identifiers").headOption.map(parseIdentifiers).getOrElse(Seq()), + license = (node \ "license").text, + vulnerabilities = vulnerabilities.map(parseVulnerability(_)), + suppressedVulnerabilities = suppressedVulnerabilities.map(parseVulnerability(_, "suppressedVulnerability")), + relatedDependencies = SerializableXml(node \ "relatedDependencies") + ) + } + + def parseEvidence(node: Node): Evidence = { + if(node.label != "evidence"){ + sys.error(s"Unexpected element for evidence: ${node.label}") + } + checkElements(node, Set("source", "name", "value")) + checkParams(node, Set("confidence", "type")) + Evidence( + source = (node \ "source").text, + name = (node \ "name").text, + value = (node \ "value").text, + confidence = node.attribute("confidence").map(_.text).get, + evidenceType = node.attribute("type").map(_.text).get + ) + } + + def parseDependencies(nodes: NodeSeq): Seq[Dependency] = nodes.map(parseDependency(_)) + + def parseXmlReport(data: Array[Byte]) = { + val xml = SecureXml.loadString(new String(data, "utf-8")) + Analysis( + scanInfo = SerializableXml((xml \ "scanInfo").head), + name = (xml \ "projectInfo" \ "name").text, + reportDate = DateTime.parse((xml \ "projectInfo" \ "reportDate").text), + dependencies = parseDependencies(xml \ "dependencies" \ "dependency").toIndexedSeq + ) + } + +} diff --git a/app/com/ysoft/odc/SecureXml.scala b/app/com/ysoft/odc/SecureXml.scala new file mode 100644 index 0000000..1a55df9 --- /dev/null +++ b/app/com/ysoft/odc/SecureXml.scala @@ -0,0 +1,17 @@ +package com.ysoft.odc + +import javax.xml.parsers.SAXParserFactory + +import scala.xml.{Elem, XML} + +// copied from https://github.com/scala/scala-xml/issues/17 and slightly modified + +object SecureXml { + def loadString(xml: String): Elem = { + val spf = SAXParserFactory.newInstance() + spf.setFeature("http://xml.org/sax/features/external-general-entities", false) + spf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true) + val saxParser = spf.newSAXParser() + XML.withSAXParser(saxParser).loadString(xml) + } +} \ No newline at end of file diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala new file mode 100644 index 0000000..44531cc --- /dev/null +++ b/app/controllers/Application.scala @@ -0,0 +1,293 @@ +package controllers + +import java.sql.BatchUpdateException + +import com.github.nscala_time.time.Imports._ +import com.google.inject.Inject +import com.google.inject.name.Named +import models._ +import play.api.Logger +import play.api.data.Forms._ +import play.api.data._ +import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} +import play.api.http.ContentTypes +import play.api.i18n.{I18nSupport, MessagesApi} +import play.api.libs.json._ +import play.api.mvc._ +import play.api.routing.JavaScriptReverseRouter +import play.twirl.api.Txt +import services.{LibrariesService, LibraryTagAssignmentsService, TagsService} +import views.html.DefaultRequest + +import scala.collection.immutable.SortedMap +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +object ApplicationFormats{ + implicit val libraryTagPairFormat = Json.format[LibraryTagPair] + implicit val libraryTagAssignmentFormat = Json.format[LibraryTagAssignment] + //implicit val libraryTypeFormat = Json.format[LibraryType] + //implicit val plainLibraryIdentifierFormat = Json.format[PlainLibraryIdentifier] + //implicit val libraryFormat = Json.format[Library] + implicit val libraryTagFormat = Json.format[LibraryTag] +} + +object export { + import ApplicationFormats._ + final case class AssignedTag(name: String, contextDependent: Boolean) + final case class TaggedLibrary(identifier: String, classified: Boolean, tags: Seq[AssignedTag]){ + def toLibrary = Library(plainLibraryIdentifier = PlainLibraryIdentifier.fromString(identifier), classified = classified) + } + final case class Export(libraryMapping: Seq[TaggedLibrary], tags: Seq[LibraryTag]) + implicit val assignedTagFormats = Json.format[AssignedTag] + implicit val taggedLibraryFormats = Json.format[TaggedLibrary] + implicit val exportFormats = Json.format[Export] +} + + +class Application @Inject() ( + reportsParser: DependencyCheckReportsParser, + reportsProcessor: DependencyCheckReportsProcessor, + projectReportsProvider: ProjectReportsProvider, + @Named("missing-GAV-exclusions") missingGAVExclusions: MissingGavExclusions, + tagsService: TagsService, + librariesService: LibrariesService, + libraryTagAssignmentsService: LibraryTagAssignmentsService, + protected val dbConfigProvider: DatabaseConfigProvider, + val messagesApi: MessagesApi, + val env: AuthEnv +) extends AuthenticatedController with HasDatabaseConfigProvider[models.profile.type]{ + + import ApplicationFormats._ + import dbConfig.driver.api._ + import models.tables.snoozesTable + import reportsProcessor.processResults + + import secureRequestConversion._ + + val dateFormatter = DateTimeFormat.forPattern("dd-MM-yyyy") + val emptySnoozeForm = Form(mapping( + "until" -> text.transform(LocalDate.parse(_, dateFormatter), (_: LocalDate).toString(dateFormatter)).verifying("Must be a date in the future", _ > LocalDate.now), + //"snoozed_object_identifier" -> text, + "reason" -> text(minLength = 3, maxLength = 255) + )(ObjectSnooze.apply)(ObjectSnooze.unapply)) + + def loadSnoozes() = { + val now = LocalDate.now + import models.jodaSupport._ + for{ + bareSnoozes <- db.run(snoozesTable.filter(_.until > now).result) : Future[Seq[(Int, Snooze)]] + snoozes = bareSnoozes.groupBy(_._2.snoozedObjectId).mapValues(ss => SnoozeInfo(emptySnoozeForm, ss.sortBy(_._2.until))).map(identity) + } yield snoozes.withDefaultValue(SnoozeInfo(emptySnoozeForm, Seq())) + } + + def purgeCache(versions: Map[String, Int], next: String) = Action { + projectReportsProvider.purgeCache(versions) + next match { + case "index" => Redirect(routes.Application.index(versions)) + case _ => Ok(Txt("CACHE PURGED")) + } + } + + def index(versions: Map[String, Int]) = ReadAction.async{ implicit req => + loadSnoozes() flatMap { snoozes => + indexPage(versions)(snoozes, securedRequestToUserAwareRequest(req)) + } + } + + def indexPage(requiredVersions: Map[String, Int])(implicit snoozes: SnoozesInfo, requestHeader: DefaultRequest) = { + val (lastRefreshTimeFuture, resultsFuture) = projectReportsProvider.resultsForVersions(requiredVersions) + processResults(resultsFuture, requiredVersions).flatMap{ case (vulnerableDependencies, allWarnings, groupedDependencies) => + Logger.debug("indexPage: Got results") + //val unclassifiedDependencies = groupedDependencies.filterNot(ds => MissingGAVExclusions.exists(_.matches(ds))).filterNot(_.identifiers.exists(_.isClassifiedInSet(classifiedSet))) + for{ + knownDependencies <- librariesService.allBase + _ = Logger.debug("indexPage: #1") + includedDependencies = groupedDependencies.filterNot(missingGAVExclusions.isExcluded) + _ = Logger.debug("indexPage: #2") + unknownDependencies = includedDependencies.flatMap(_.identifiers.flatMap(_.toLibraryIdentifierOption)).toSet -- knownDependencies.map(_.plainLibraryIdentifier).toSet + _ = Logger.debug("indexPage: #3") + _ <- librariesService.insertMany(unknownDependencies.map(Library(_, classified = false))) + _ = Logger.debug("indexPage: #3") + unclassifiedDependencies <- librariesService.unclassified + _ = Logger.debug("indexPage: #4") + allTags <- tagsService.all + _ = Logger.debug("indexPage: #6") + allTagsMap = allTags.toMap + _ = Logger.debug("indexPage: #7") + tagsWithWarning = allTags.collect(Function.unlift{case (id, t: LibraryTag) => t.warningOrder.map(_ => (id, t))}).sortBy(_._2.warningOrder) + _ = Logger.debug("indexPage: #8") + librariesForTagsWithWarningUnsorted <- librariesService.librariesForTags(tagsWithWarning.map(_._1)) + _ = Logger.debug("indexPage: #9") + librariesForTagsWithWarning = SortedMap(librariesForTagsWithWarningUnsorted.groupBy(_._1).toSeq.map{case (tagId, lr) => (tagId, allTagsMap(tagId)) -> lr.map(_._2) } : _*)(Ordering.by(t => (t._2.warningOrder, t._1))) + _ = Logger.debug("indexPage: #10") + relatedDependenciesTags <- librariesService.byTags(unclassifiedDependencies.map(_._1).toSet ++ librariesForTagsWithWarning.values.flatten.map(_._1).toSet) + _ = Logger.debug("indexPage: #11") + lastRefreshTime <- lastRefreshTimeFuture + } yield { + Logger.debug("indexPage: Got all ingredients") + Ok(views.html.index( + vulnerableDependencies = vulnerableDependencies, + warnings = allWarnings, + librariesForTagsWithWarning = librariesForTagsWithWarning, + unclassifiedDependencies = unclassifiedDependencies, + groupedDependencies = groupedDependencies, + dependenciesForLibraries = groupedDependencies.flatMap(group => + group.identifiers.flatMap(_.toLibraryIdentifierOption).map(_ -> group) + ).groupBy(_._1).mapValues(_.map(_._2).toSet).map(identity), + allTags = allTags, + relatedDependenciesTags = relatedDependenciesTags, + lastRefreshTime = lastRefreshTime, + versions = requiredVersions + )) + } + } recover { + case e: BatchUpdateException => + throw e.getNextException + } + } + + implicit class AddAdjustToMap[K, V](m: Map[K, V]){ + def adjust(k: K)(f: V => V) = m.updated(k, f(m(k))) + } + + def snooze(id: String, versions: Map[String, Int]) = AdminAction.async { implicit req => + loadSnoozes().flatMap{ loadedSnoozes => + val snoozes = loadedSnoozes.adjust(id){_.adjustForm(_.bindFromRequest()(req))} + snoozes(id).form.fold( + f => indexPage(Map())(snoozes, securedRequestToUserAwareRequest(req)), + snooze => for { + _ <- db.run(snoozesTable.map(_.base) += snooze.toSnooze(id)) + } yield Redirect(routes.Application.index(versions).withFragment(id)) + ) + } + } + + def unsnooze(snoozeId: Int, versions: Map[String, Int]) = AdminAction.async { implicit req => + (db.run(snoozesTable.filter(_.id === snoozeId).map(_.base).result).map(_.headOption): Future[Option[Snooze]]).flatMap { + case Some(snooze) => + for(_ <- db.run(snoozesTable.filter(_.id === snoozeId).delete)) yield Redirect(routes.Application.index(versions).withFragment(snooze.snoozedObjectId)) + case None => Future.successful(NotFound(Txt("Unknown snoozeId"))) + } + } + + // TODO: move import/export to a separate controller + def tagsExport = Action.async { + import export._ + for{ + tags <- tagsService.all.map(_.toMap) + lta <- libraryTagAssignmentsService.byLibrary + libs <- librariesService.touched(lta.keySet) + } yield { + val libraryMapping = (libs: Seq[(Int, Library)]).sortBy(_._2.plainLibraryIdentifier.toString).map { case (id, l) => + val assignments: Seq[LibraryTagAssignment] = lta(id) + TaggedLibrary( + identifier = s"${l.plainLibraryIdentifier}", + classified = l.classified, + tags = assignments.map(a => AssignedTag(name = tags(a.tagId).name, contextDependent = a.contextDependent)).sortBy(_.name.toLowerCase) + ) + } + Ok(Json.prettyPrint(Json.toJson( + Export(libraryMapping = libraryMapping, tags = tags.values.toSeq.sortBy(_.name.toLowerCase)) + ))).as(ContentTypes.JSON) + } + } + + val tagsImportForm = Form(mapping("data" -> text)(identity)(Some(_))) + + def tagsImport = AdminAction { implicit req => + Ok(views.html.tagsImport(tagsImportForm)) + } + + def tagsImportAction = AdminAction.async { implicit req => + tagsImportForm.bindFromRequest()(req).fold( + formWithErrors => ???, + data => + export.exportFormats.reads(Json.parse(data)).fold( + invalid => Future.successful(BadRequest(Txt("ERROR: "+invalid))), + data => { + def importTags() = tagsService.insertMany(data.tags) + def getTagsByName(): Future[Map[String, Int]] = tagsService.all.map(_.groupBy(_._2.name).mapValues { case Seq((id, _)) => id }.map(identity)) + def importLibraries(): Future[Unit] = Future.sequence( + data.libraryMapping.map{ taggedLibrary => + librariesService.insert(taggedLibrary.toLibrary).flatMap{ libraryId => + importLibraryTagAssignment(libraryId, taggedLibrary) + } + } + ).map( (x: Seq[Unit]) => ()) // I don't care about the result + def importLibraryTagAssignment(libraryId: Int, taggedLibrary: export.TaggedLibrary): Future[Unit] = getTagsByName().flatMap { tagIdsByName => + Future.sequence(taggedLibrary.tags.map{ assignedTag => + val tagId = tagIdsByName(assignedTag.name) + libraryTagAssignmentsService.insert(LibraryTagAssignment(LibraryTagPair(libraryId = libraryId, tagId = tagId), assignedTag.contextDependent)).map(_ => ()) + }).map( (x: Seq[Unit]) => ()) // I don't care about the result + } + for { + _ <- importTags() + _ <- importLibraries() + } yield Ok(Txt("OK")) + } + ) + + ) + } + + def dependencies(requiredClassification: Option[Boolean], requiredTags: Seq[Int], noTag: Boolean) = ReadAction.async { implicit request => + val requiredTagsSet = requiredTags.toSet + for{ + selectedDependencies <- db.run(librariesService.filtered(requiredClassification = requiredClassification, requiredTagsOption = if(noTag) None else Some(requiredTagsSet)).result) + dependencyTags <- librariesService.byTags(selectedDependencies.map(_._1).toSet) + allTags <- tagsService.all + }yield{ + Ok(views.html.dependencies( + requiredClassification = requiredClassification, + selectedDependencies = selectedDependencies, + allTags = allTags, + dependencyTags = dependencyTags, + requiredTagSet = requiredTagsSet, + noTag = noTag, + tagsLink = (newTags: Set[Int]) => routes.Application.dependencies(requiredClassification, newTags.toSeq.sorted, noTag), + noTagLink = newNoTag => routes.Application.dependencies(requiredClassification, requiredTagsSet.toSeq.sorted, newNoTag), + classificationLink = newClassification => routes.Application.dependencies(newClassification, requiredTagsSet.toSeq.sorted, noTag) + )) + } + } + + def removeTag() = AdminAction.async(BodyParsers.parse.json) { request => + request.body.validate[LibraryTagPair].fold( + err => Future.successful(BadRequest(Txt(err.toString()))), + libraryTagPair => for(_ <- libraryTagAssignmentsService.remove(libraryTagPair)) yield Ok(Txt("OK")) + ) + } + + def addTag() = AdminAction.async(BodyParsers.parse.json) { request => + request.body.validate[LibraryTagAssignment].fold( + err => Future.successful(BadRequest(Txt(err.toString()))), + tagAssignment => for(_ <- libraryTagAssignmentsService.insert(tagAssignment)) yield {Ok(Txt("OK"))} + ) + } + + def setClassified(classified: Boolean) = AdminAction.async(BodyParsers.parse.json) {request => + val libraryId = request.body.as[Int] + for(_ <- librariesService.setClassified(libraryId, classified)) yield Ok(Txt("OK")) + } + + def javascriptRoutes = Action { implicit request => + Ok( + JavaScriptReverseRouter("Routes")( + routes.javascript.Application.setClassified, + routes.javascript.Application.addTag + ) + ).as("text/javascript") + } + + def testHttps(allowRedirect: Boolean) = Action { Ok(Txt(if(allowRedirect) + """ + |(function(){ + | var newUrl = window.location.href.replace(/^http:/, "https:"); + | if(newUrl != window.location.href){ + | window.location.replace(newUrl); + | } + |})(); + |""".stripMargin else "")).withHeaders("Content-type" -> "text/javascript; charset=utf-8") } + +} diff --git a/app/controllers/AuthController.scala b/app/controllers/AuthController.scala new file mode 100644 index 0000000..73f96ee --- /dev/null +++ b/app/controllers/AuthController.scala @@ -0,0 +1,64 @@ +package controllers + +import javax.inject.Inject + +import _root_.services.CredentialsVerificationService +import com.mohiva.play.silhouette.api._ +import com.mohiva.play.silhouette.api.util.Clock +import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator +import models.User +import play.api.data.Form +import play.api.data.Forms._ +import play.api.i18n.{Messages, MessagesApi} +import play.api.libs.concurrent.Execution.Implicits._ + +import scala.concurrent.Future + +final case class LoginRequest(username: String, password: String, rememberMe: Boolean) + +class AuthController @Inject() ( + val messagesApi: MessagesApi, + val env: Environment[User, CookieAuthenticator], + clock: Clock, + credentialsVerificationService: CredentialsVerificationService +) extends AuthenticatedController { + + val signInForm = Form(mapping( + "username" -> nonEmptyText, + "password" -> nonEmptyText, + "rememberMe" -> boolean + )(LoginRequest.apply)(LoginRequest.unapply)) + + def signIn = UserAwareAction { implicit request => + request.identity match { + case Some(user) => Redirect(routes.Application.index(Map())) + case None => Ok(views.html.auth.signIn(signInForm/*, socialProviderRegistry*/)) + } + } + + def authenticate() = UserAwareAction.async { implicit request => + signInForm.bindFromRequest().fold( + formWithErrors => Future.successful(BadRequest(views.html.auth.signIn(formWithErrors/*, socialProviderRegistry*/))), + loginRequest => { + credentialsVerificationService.verifyCredentials(loginRequest.username, loginRequest.password).flatMap{ + case true => + val loginInfo: LoginInfo = LoginInfo(providerID = "credentials-verification", providerKey = loginRequest.username) + val user: User = User(username = loginRequest.username) + env.authenticatorService.create(loginInfo) flatMap { authenticator => + env.eventBus.publish(LoginEvent(user, request, implicitly[Messages])) + env.authenticatorService.init(authenticator).flatMap(cookie => + env.authenticatorService.embed(cookie.copy(secure = request.secure), Redirect(routes.Application.index(Map()))) + ) + } + case false => Future.successful(Redirect(routes.AuthController.signIn()).flashing("error" -> Messages("invalid.credentials"))) + } + } + ) + } + + def signOut = SecuredAction.async { implicit request => + val result = Redirect(routes.Application.index(Map())) + env.eventBus.publish(LogoutEvent(request.identity, request, request2Messages)) + env.authenticatorService.discard(request.authenticator, result) + } +} diff --git a/app/controllers/AuthenticatedController.scala b/app/controllers/AuthenticatedController.scala new file mode 100644 index 0000000..6aca545 --- /dev/null +++ b/app/controllers/AuthenticatedController.scala @@ -0,0 +1,31 @@ +package controllers + +import com.mohiva.play.silhouette.api.Silhouette +import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator +import models.User +import play.api.mvc.{Result, RequestHeader, Results} +import views.html.DefaultRequest + +import scala.concurrent.Future +import scala.language.implicitConversions + +trait AuthenticatedControllerLowPriorityImplicits[T, C]{ + self: AuthenticatedController => + + protected object secureRequestConversion{ + implicit def securedRequestToUserAwareRequest(implicit req: SecuredRequest[_]): DefaultRequest = UserAwareRequest(Some(req.identity), authenticator = Some(req.authenticator), req.request) + } +} + +abstract class AuthenticatedController extends Silhouette[User, CookieAuthenticator] with AuthenticatedControllerLowPriorityImplicits[User, CookieAuthenticator]{ + + + override protected def onNotAuthenticated(request: RequestHeader): Option[Future[Result]] = Some(Future.successful(Redirect(routes.AuthController.signIn()))) + + object ReadAction extends SecuredActionBuilder with Results { + + } + + def AdminAction: SecuredActionBuilder = ??? + +} \ No newline at end of file diff --git a/app/controllers/DependencyCheckReportsParser.scala b/app/controllers/DependencyCheckReportsParser.scala new file mode 100644 index 0000000..e596701 --- /dev/null +++ b/app/controllers/DependencyCheckReportsParser.scala @@ -0,0 +1,143 @@ +package controllers + +import java.net.URLEncoder + +import com.google.inject.Inject +import com.ysoft.odc._ +import controllers.DependencyCheckReportsParser.Result +import models.PlainLibraryIdentifier +import play.api.Logger +import play.api.cache.CacheApi +import play.twirl.api.Html + +import scala.util.{Failure, Success, Try} + +sealed trait Filter{ + def selector: Option[String] + def subReports(r: Result): Option[Result] + def filters: Boolean + def descriptionHtml: Html + def descriptionText: String +} +private final case class ProjectFilter(project: ReportInfo) extends Filter{ + override def filters: Boolean = true + override def descriptionHtml: Html = views.html.filters.project(project) + override def descriptionText: String = s"project ${friendlyProjectName(project)}" + override def subReports(r: Result): Option[Result] = { + @inline def reportInfo = project + def f[T](m: Map[ReportInfo, T]): Map[String, T] = ( + if(reportInfo.subprojectNameOption.isEmpty) m.filter(_._1.projectId == project.projectId) else m.get(reportInfo).fold(Map.empty[ReportInfo, T])(x => Map(reportInfo -> x)) + ).map{case (k, v) => k.fullId -> v} + val newFlatReports = f(r.flatReports) + val newFailedAnalysises = f(r.failedAnalysises) + if(newFlatReports.isEmpty && newFailedAnalysises.isEmpty) None + else Some(Result(bareFlatReports = newFlatReports, bareFailedAnalysises = newFailedAnalysises, projects = r.projects)) + } + override def selector = Some(s"project:${project.fullId}") +} +private final case class TeamFilter(team: Team) extends Filter{ + override def filters: Boolean = true + override def subReports(r: Result): Option[Result] = { + + val reportInfoByFriendlyProjectName = r.projectsReportInfo.ungroupedReportsInfo.map(ri => friendlyProjectName(ri) -> ri).toSeq.groupBy(_._1).mapValues{ + case Seq((_, ri)) => ri + case other => sys.error("some duplicate value: "+other) + }.map(identity) + val reportInfos = team.projectNames.map(reportInfoByFriendlyProjectName) + def submap[T](m: Map[String, T]) = reportInfos.toSeq.flatMap(ri => m.get(ri.fullId).map(ri.fullId -> _) ).toMap + Some(Result( + bareFlatReports = submap(r.bareFlatReports), + bareFailedAnalysises = submap(r.bareFailedAnalysises), + projects = r.projects + )) + } + override def descriptionHtml: Html = views.html.filters.team(team.id) + override def descriptionText: String = s"team ${team.name}" + override def selector = Some(s"team:${team.id}") +} +object NoFilter extends Filter{ + override def filters: Boolean = false + override val descriptionHtml: Html = views.html.filters.all() + override def descriptionText: String = "all projects" + override def subReports(r: Result): Option[Result] = Some(r) + override def selector: Option[String] = None +} +private final case class BadFilter(pattern: String) extends Filter{ + override def filters: Boolean = true + override def subReports(r: Result): Option[Result] = None + override def descriptionHtml: Html = Html("bad filter") + override def descriptionText: String = "bad filter" + override def selector: Option[String] = Some(pattern) +} + +object DependencyCheckReportsParser{ + final case class ResultWithSelection(result: Result, projectsWithSelection: ProjectsWithSelection) + final case class Result(bareFlatReports: Map[String, Analysis], bareFailedAnalysises: Map[String, Throwable], projects: Projects){ + lazy val projectsReportInfo = new ProjectsWithReports(projects, bareFlatReports.keySet ++ bareFailedAnalysises.keySet) + lazy val flatReports: Map[ReportInfo, Analysis] = bareFlatReports.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v} + lazy val failedAnalysises: Map[ReportInfo, Throwable] = bareFailedAnalysises.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v} + lazy val allDependencies = flatReports.toSeq.flatMap(r => r._2.dependencies.map(_ -> r._1)) + lazy val groupedDependencies = allDependencies.groupBy(_._1.hashes).values.map(GroupedDependency(_)).toSeq + lazy val groupedDependenciesByPlainLibraryIdentifier: Map[PlainLibraryIdentifier, Set[GroupedDependency]] = + groupedDependencies.toSet.flatMap((grDep: GroupedDependency) => grDep.plainLibraryIdentifiers.map(_ -> grDep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity) + lazy val vulnerableDependencies = groupedDependencies.filter(_.vulnerabilities.nonEmpty) + + private val ProjectSelectorPattern = """^project:(.*)$""".r + private val TeamSelectorPattern = """^team:(.*)$""".r + + private def parseFilter(filter: String): Filter = filter match { + case ProjectSelectorPattern(project) => ProjectFilter(projectsReportInfo.reportIdToReportInfo(project)) + case TeamSelectorPattern(team) => TeamFilter(projects.teamById(team)) + case other => BadFilter(other) + } + + def selection(selectorOption: Option[String]): Option[ResultWithSelection] = { + val filter = selectorOption.map(parseFilter).getOrElse(NoFilter) + filter.subReports(this).map{ result => + ResultWithSelection( + result = result, + projectsWithSelection = ProjectsWithSelection(filter = filter, projectsWithReports = projectsReportInfo, teams = projects.teamSet) + ) + } + } + + } +} + +final class DependencyCheckReportsParser @Inject() (cache: CacheApi, projects: Projects) { + + def parseReports(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)]) = { + val rid = math.random.toString // for logging + @volatile var parseFailedForSomeAnalysis = false + val deepReportsTriesIterable: Iterable[Map[String, Try[Analysis]]] = for((k, (build, data, log)) <- successfulResults) yield { + Logger.debug(data.flatFilesWithPrefix(s"$k/").keySet.toSeq.sorted.toString) + val flat = data.flatFilesWithPrefix(s"$k/") + (for((k, v) <- flat.par) yield { + val analysisKey = URLEncoder.encode(s"analysis/parsedXml/${build.buildResultKey}/${k}", "utf-8") + Logger.debug(s"[$rid] analysisKey: $analysisKey") + val analysisTry = cache.getOrElse(analysisKey)(Try{OdcParser.parseXmlReport(v)}) + analysisTry match{ + case Success(e) => // nothing + case Failure(e) => + if(!parseFailedForSomeAnalysis){ + Logger.error(s"[$rid] Cannot parse $k: ${new String(v, "utf-8")}", e) + parseFailedForSomeAnalysis = true + } + } + k -> analysisTry + }).seq + } + val deepReportsAndFailuresIterable = deepReportsTriesIterable.map { reports => + val (successfulReportsTries, failedReportsTries) = reports.partition(_._2.isSuccess) + val successfulReports = successfulReportsTries.mapValues(_.asInstanceOf[Success[Analysis]].value).map(identity) + val failedReports = failedReportsTries.mapValues(_.asInstanceOf[Failure[Analysis]].exception).map(identity) + (successfulReports, failedReports) + } + val deepSuccessfulReports = deepReportsAndFailuresIterable.map(_._1).toSeq + val failedAnalysises = deepReportsAndFailuresIterable.map(_._2).toSeq.flatten.toMap + val flatReports = deepSuccessfulReports.flatten.toMap + Logger.debug(s"[$rid] parse finished") + Result(flatReports, failedAnalysises, projects) + } + +} diff --git a/app/controllers/DependencyCheckReportsProcessor.scala b/app/controllers/DependencyCheckReportsProcessor.scala new file mode 100644 index 0000000..2024bb8 --- /dev/null +++ b/app/controllers/DependencyCheckReportsProcessor.scala @@ -0,0 +1,106 @@ +package controllers + +import com.github.nscala_time.time.Imports._ +import com.google.inject.Inject +import com.google.inject.name.Named +import com.ysoft.odc.Checks._ +import com.ysoft.odc._ +import org.joda.time.DateTimeConstants +import play.api.Logger +import play.api.i18n.{I18nSupport, MessagesApi} +import play.api.mvc.RequestHeader +import play.twirl.api.Html +import views.html.DefaultRequest + +import scala.concurrent.{ExecutionContext, Future} + +final case class MissingGavExclusions(exclusionsSet: Set[Exclusion]){ + def isExcluded(groupedDependency: GroupedDependency) = exclusionsSet.exists(_.matches(groupedDependency)) +} + +final class DependencyCheckReportsProcessor @Inject() ( + @Named("bamboo-server-url") val server: String, + dependencyCheckReportsParser: DependencyCheckReportsParser, + @Named("missing-GAV-exclusions") missingGAVExclusions: MissingGavExclusions, + val messagesApi: MessagesApi + ) extends I18nSupport { + + private def parseDateTime(dt: String): DateTime = { + if(dt.forall(_.isDigit)){ + new DateTime(dt.toLong) // TODO: timezone (I don't care much, though) + }else{ + val formatter = DateTimeFormat.forPattern("dd/MM/yyyy HH:mm:ss") // TODO: timezone (I don't care much, though) + formatter.parseDateTime(dt) + } + } + + @deprecated("use HTML output instead", "SNAPSHOT") private val showDependencies: (Seq[GroupedDependency]) => Seq[String] = { + _.map { s => + s.dependencies.map { case (dep, projects) => s"${dep.fileName} @ ${projects.toSeq.sorted.map(friendlyProjectName).mkString(", ")}" }.mkString(", ") + " " + s.hashes + } + } + + + def processResults( + resultsFuture: Future[(Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable])], + requiredVersions: Map[String, Int] + )(implicit requestHeader: DefaultRequest, snoozesInfo: SnoozesInfo, executionContext: ExecutionContext) = try{ + for((successfulResults, failedResults) <- resultsFuture) yield{ + val reportResult = dependencyCheckReportsParser.parseReports(successfulResults) + import reportResult.{allDependencies, failedAnalysises, flatReports, groupedDependencies, vulnerableDependencies} + val now = DateTime.now + val oldReportThreshold = now - 1.day + val cveTimestampThreshold = now - (if(now.dayOfWeek().get == DateTimeConstants.MONDAY) 4.days else 2.days ) + val ScanChecks: Seq[Map[ReportInfo, Analysis] => Option[Warning]] = Seq( + differentValues("scan infos", "scan-info", WarningSeverity.Warning)(_.groupBy(_._2.scanInfo).mapValues(_.keys.toIndexedSeq.sorted)), + badValues("old-reports", "old reports", WarningSeverity.Warning)((_, a) => if(a.reportDate < oldReportThreshold) Some(Html(a.reportDate.toString)) else None), + badValues("bad-cve-data", "old or no CVE data", WarningSeverity.Warning){(_, analysis) => + (analysis.scanInfo.xml \\ "timestamp").map(_.text).filterNot(_ == "").map(parseDateTime) match { + case Seq() => Some(Html("no data")) + case timestamps => + val newestTimestamp = timestamps.max + val oldestTimestamp = timestamps.min + if(newestTimestamp < cveTimestampThreshold) Some(Html(newestTimestamp.toString)) + else None + } + } + ) + val GroupedDependenciesChecks = Seq[Seq[GroupedDependency] => Option[Warning]]( + badGroupedDependencies("unidentified-dependencies", "unidentified dependencies", WarningSeverity.Info)(_.filter(_.dependencies.exists(_._1.identifiers.isEmpty)))(show = showDependencies, exclusions = missingGAVExclusions.exclusionsSet), + badGroupedDependencies("different-identifier-sets", "different identifier sets", WarningSeverity.Info)(_.filter(_.dependencies.groupBy(_._1.identifiers).size > 1).toIndexedSeq)(), + badGroupedDependencies("different-evidence", "different evidence", WarningSeverity.Info)(_.filter(_.dependencies.groupBy(_._1.evidenceCollected).size > 1).toIndexedSeq)(show = x => Some(views.html.warnings.groupedDependencies(x))), + badGroupedDependencies("missing-gav", "missing GAV", WarningSeverity.Info)(_.filter(_.identifiers.filter(_.identifierType == "maven").isEmpty))(show = showDependencies, exclusions = missingGAVExclusions.exclusionsSet) + ) + + val unknownIdentifierTypes = allDependencies.flatMap(_._1.identifiers.map(_.identifierType)).toSet -- Set("maven", "cpe") + val failedReports = successfulResults.filter(x => x._2._1.state != "Successful" || x._2._1.buildState != "Successful") + val extraWarnings = Seq[Option[Warning]]( + if(failedReports.size > 0) Some(IdentifiedWarning("failed-reports", views.html.warnings.failedReports(failedReports.values.map{case (b, _ ,_) => b}.toSet, server), WarningSeverity.Error)) else None, + if(unknownIdentifierTypes.size > 0) Some(IdentifiedWarning("unknown-identifier-types", views.html.warnings.unknownIdentifierType(unknownIdentifierTypes), WarningSeverity.Info)) else None, + { + val emptyResults = successfulResults.filter{case (k, (_, dir, _)) => dir.flatFiles.size < 1} + if(emptyResults.nonEmpty) Some(IdentifiedWarning("empty-results", views.html.warnings.emptyResults(emptyResults.values.map{case (build, _, _) => build}.toSeq, server), WarningSeverity.Warning)) else None + }, + { + val resultsWithErrorMessages = successfulResults.filter{case (k, (_, _, log)) => log.dataString.lines.exists(l => (l.toLowerCase startsWith "error") || (l.toLowerCase contains "[error]"))} + if(resultsWithErrorMessages.nonEmpty) Some(IdentifiedWarning("results-with-error-messages", views.html.warnings.resultsWithErrorMessages(resultsWithErrorMessages.values.map{case (build, _, _) => build}.toSeq, server), WarningSeverity.Error)) else None + }, + if(failedResults.isEmpty) None else Some(IdentifiedWarning("failed-results", views.html.warnings.failedResults(failedResults), WarningSeverity.Error)), + if(requiredVersions.isEmpty) None else Some(IdentifiedWarning("required-versions", views.html.warnings.textWarning("You have manually requested results for some older version."), WarningSeverity.Warning)), + if(failedAnalysises.isEmpty) None else Some(IdentifiedWarning("failed-analysises", views.html.warnings.textWarning(s"Some reports failed to parse: ${failedAnalysises.keySet}"), WarningSeverity.Error)) + ).flatten + + val scanWarnings = ScanChecks.flatMap(_(flatReports)) + val groupedDependenciesWarnings = GroupedDependenciesChecks.flatMap(_(groupedDependencies)) + val allWarnings = scanWarnings ++ groupedDependenciesWarnings ++ extraWarnings + + // TODO: log analysis + // TODO: related dependencies + (vulnerableDependencies, allWarnings, groupedDependencies) + } + }finally{ + Logger.debug("Reports processed") + } + + +} diff --git a/app/controllers/ProjectReportsProvider.scala b/app/controllers/ProjectReportsProvider.scala new file mode 100644 index 0000000..e0af52c --- /dev/null +++ b/app/controllers/ProjectReportsProvider.scala @@ -0,0 +1,40 @@ +package controllers + +import com.github.nscala_time.time.Imports._ +import com.google.inject.Inject +import com.ysoft.odc.Downloader +import play.api.cache.CacheApi + +import scala.concurrent.{ExecutionContext, Future} +import scala.reflect.ClassTag +import scala.util.Success + +class ProjectReportsProvider @Inject() ( + bambooDownloader: Downloader, + cache: CacheApi, + projects: Projects + )(implicit executionContext: ExecutionContext){ + + private def bambooCacheKey(versions: Map[String, Int]) = "bamboo/results/" + versions.toSeq.sorted.map{case (k, v) => k.getBytes("utf-8").mkString("-") + ":" + v}.mkString("|") + + def purgeCache(versions: Map[String, Int]) = cache.remove(bambooCacheKey(versions)) + + private def getOrElseFuture[T: ClassTag] + (name: String, expiration: scala.concurrent.duration.Duration = scala.concurrent.duration.Duration.Inf) + (f: => Future[T]) + (implicit executionContext: ExecutionContext): Future[T] = + { + cache.get[T](name).map(Future.successful).getOrElse( + f.andThen{ + case Success(value) =>cache.set(name, value, expiration) + } + ) + } + + def resultsForVersions(versions: Map[String, Int]) = { + def get = {val time = DateTime.now; bambooDownloader.downloadProjectReports(projects.projectSet, versions).map(time -> _)} + val allFuture = getOrElseFuture(bambooCacheKey(versions)){println("CACHE MISS"); get} + (allFuture.map(_._1), allFuture.map(_._2)) + } + +} diff --git a/app/controllers/Projects.scala b/app/controllers/Projects.scala new file mode 100644 index 0000000..4a77f6e --- /dev/null +++ b/app/controllers/Projects.scala @@ -0,0 +1,52 @@ +package controllers + +import javax.inject.Inject + +import play.api.Configuration + +class Projects @Inject() (configuration: Configuration) { + import scala.collection.JavaConversions._ + val projectMap = { + val projectsConfig = configuration.getObject("yssdc.projects").getOrElse(sys.error("yssdc.projects is not set")).toConfig + projectsConfig.entrySet().map( k => k.getKey -> projectsConfig.getString(k.getKey)).toMap + } + val projectSet = projectMap.keySet + val teamIdSet = configuration.getStringSeq("yssdc.teams").getOrElse(sys.error("yssdc.teams is not set")).map(TeamId).toSet + private val teamsByIds = teamIdSet.map(t => t.id -> t).toMap + val teamLeaders = { + import scala.collection.JavaConversions._ + configuration.getObject("yssdc.teamLeaders").getOrElse(sys.error("yssdc.teamLeaders is not set")).map{case(k, v) => + TeamId(k) -> v.unwrapped().asInstanceOf[String] + } + } + { + val extraTeams = teamLeaders.keySet -- teamIdSet + if(extraTeams.nonEmpty){ + sys.error(s"Some unexpected teams: $extraTeams") + } + } + + def existingTeamId(s: String): TeamId = teamsByIds(s) + + val projectToTeams = configuration.getObject("yssdc.projectsToTeams").get.mapValues{_.unwrapped().asInstanceOf[java.util.List[String]].map(c => + existingTeamId(c) + ).toSet}.map(identity) + + val projectAndTeams = projectToTeams.toSeq.flatMap{case (project, teams) => teams.map(team => (project, team))} + + val teamsToProjects = projectAndTeams.groupBy(_._2).mapValues(_.map(_._1).toSet).map(identity) + + val teamsById: Map[String, Team] = for{ + (team, projectNames) <- teamsToProjects + } yield team.id -> Team( + id = team.id, + name = team.name, + leader = teamLeaders(team), + projectNames = projectNames + ) + + def teamById(id: String) = teamsById(id) + + def teamSet = teamsById.values.toSet + +} diff --git a/app/controllers/ProjectsWithReports.scala b/app/controllers/ProjectsWithReports.scala new file mode 100644 index 0000000..20b3148 --- /dev/null +++ b/app/controllers/ProjectsWithReports.scala @@ -0,0 +1,58 @@ +package controllers + +final case class ReportInfo( + projectId: String, + projectName: String, + fullId: String, + subprojectNameOption: Option[String] +) extends Ordered[ReportInfo] { + + import scala.math.Ordered.orderingToOrdered + + override def compare(that: ReportInfo): Int = ((projectName, subprojectNameOption, fullId)) compare ((that.projectName, that.subprojectNameOption, that.fullId)) + + // It seems to be a good idea to have a custom equals and hashCode for performance reasons + + + override def equals(other: Any): Boolean = other match { + case other: ReportInfo => fullId == other.fullId + case _ => false + } + + override def hashCode(): Int = 517+fullId.hashCode + +} + +object ProjectsWithReports{ + + private val RestMessBeginRegexp = """^/Report results-XML/""".r + + private val RestMessEndRegexp = """/(target/)?dependency-check-report\.xml$""".r + +} + +class ProjectsWithReports (val projects: Projects, val reports: Set[String]) { + + import ProjectsWithReports._ + + val reportIdToReportInfo = { + val reportsMap = reports.map{ unfriendlyName => + val (baseName, theRest) = unfriendlyName.span(_ != '/') + val removeLeadingMess = RestMessBeginRegexp.replaceAllIn(_: String, "") + val removeTrailingMess = RestMessEndRegexp.replaceAllIn(_: String, "") + val removeMess = removeLeadingMess andThen removeTrailingMess + val subProjectOption = Some(removeMess(theRest)).filter(_ != "") + subProjectOption.fold(baseName)(baseName+"/"+_) + unfriendlyName -> ReportInfo( + projectId = baseName, + fullId = unfriendlyName, + projectName = projects.projectMap(baseName), + subprojectNameOption = subProjectOption + ) + }.toMap + reportsMap ++ reportsMap.values.map(r => r.projectId -> ReportInfo(projectId = r.projectId, fullId = r.projectId, subprojectNameOption = None, projectName = r.projectName)) + } + + val ungroupedReportsInfo = reportIdToReportInfo.values.toSet + +} diff --git a/app/controllers/ProjectsWithSelection.scala b/app/controllers/ProjectsWithSelection.scala new file mode 100644 index 0000000..ee2a62e --- /dev/null +++ b/app/controllers/ProjectsWithSelection.scala @@ -0,0 +1,14 @@ +package controllers + +final case class TeamId(id: String) extends AnyVal { + def name = id +} + +final case class Team(id: String, name: String, leader: String, projectNames: Set[String]) + +// TODO: rename to something more sane. It is maybe rather FilteringData now. +final case class ProjectsWithSelection(filter: Filter, projectsWithReports: ProjectsWithReports, teams: Set[Team]) { + def isProjectSpecified: Boolean = filter.filters + def selectorString = filter.selector + def projectNameText: String = filter.descriptionText +} diff --git a/app/controllers/Statistics.scala b/app/controllers/Statistics.scala new file mode 100644 index 0000000..fe05ce8 --- /dev/null +++ b/app/controllers/Statistics.scala @@ -0,0 +1,226 @@ +package controllers + +import com.github.nscala_time.time.Imports._ +import com.google.inject.Inject +import com.google.inject.name.Named +import com.ysoft.odc.{ArtifactFile, ArtifactItem} +import models.{Library, LibraryTag} +import org.joda.time.DateTime +import play.api.i18n.MessagesApi +import play.twirl.api.Txt +import services.{LibrariesService, LibraryTagAssignmentsService, OdcService, TagsService} +import views.html.DefaultRequest + +import scala.concurrent.{ExecutionContext, Future} + +object Statistics{ + case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency]){ + def vulnerableRatio = vulnerableDependencies.size.toDouble / dependencies.size.toDouble + lazy val vulnerabilities: Set[Vulnerability] = dependencies.flatMap(_.vulnerabilities) + lazy val vulnerabilitiesToDependencies: Map[Vulnerability, Set[GroupedDependency]] = vulnerableDependencies.flatMap(dep => + dep.vulnerabilities.map(vuln => (vuln, dep)) + ).groupBy(_._1).mapValues(_.map(_._2)).map(identity) + vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity) + vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity) + lazy val vulnerableDependencies = dependencies.filter(_.isVulnerable) + lazy val (dependenciesWithCpe, dependenciesWithoutCpe) = dependencies.partition(_.hasCpe) + lazy val cpeRatio = dependenciesWithCpe.size.toDouble / dependencies.size.toDouble + lazy val weaknesses = vulnerabilities.flatMap(_.cweOption) + lazy val weaknessesFrequency = computeWeaknessesFrequency(vulnerabilities) + } + case class TagStatistics(tagRecord: (Int, LibraryTag), stats: LibDepStatistics){ + def tag: LibraryTag = tagRecord._2 + def tagId: Int = tagRecord._1 + } + + def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0) + +} + +import controllers.Statistics._ + +class Statistics @Inject() ( + reportsParser: DependencyCheckReportsParser, + reportsProcessor: DependencyCheckReportsProcessor, + projectReportsProvider: ProjectReportsProvider, + dependencyCheckReportsParser: DependencyCheckReportsParser, + librariesService: LibrariesService, + tagsService: TagsService, + odcService: OdcService, + libraryTagAssignmentsService: LibraryTagAssignmentsService, + @Named("missing-GAV-exclusions") missingGAVExclusions: MissingGavExclusions, + projects: Projects, + val env: AuthEnv +)(implicit val messagesApi: MessagesApi, executionContext: ExecutionContext) extends AuthenticatedController { + + private val versions = Map[String, Int]() + + private def notFound()(implicit req: DefaultRequest) = { + NotFound(views.html.defaultpages.notFound("GET", req.uri)) + } + + import secureRequestConversion._ + + + private def select(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], selectorOption: Option[String]) = dependencyCheckReportsParser.parseReports(successfulResults).selection(selectorOption) + + def searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String]) = ReadAction.async{ implicit req => + if(versionlessCpes.isEmpty){ + Future.successful(notFound()) + }else{ + val now = DateTime.now() + val oldDataThreshold = 2.days + val lastDbUpdateFuture = odcService.loadLastDbUpdate() + val isOldFuture = lastDbUpdateFuture.map{ lastUpdate => now - oldDataThreshold > lastUpdate} + versionOption match { + case Some(version) => + for { + res1 <- Future.traverse(versionlessCpes) { versionlessCpe => odcService.findRelevantCpes(versionlessCpe, version) } + vulnIds = res1.flatten.map(_.vulnerabilityId).toSet + vulns <- Future.traverse(vulnIds)(id => odcService.getVulnerabilityDetails(id).map(_.get)) + isOld <- isOldFuture + } yield Ok(views.html.statistics.vulnerabilitiesForLibrary( + vulnsAndVersionOption = Some((vulns, version)), + cpes = versionlessCpes, + isDbOld = isOld + )) + case None => + for(isOld <- isOldFuture) yield Ok(views.html.statistics.vulnerabilitiesForLibrary( + vulnsAndVersionOption = None, + cpes = versionlessCpes, + isDbOld = isOld + )) + } + } + } + + def basic(projectOption: Option[String]) = ReadAction.async{ implicit req => + val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) + resultsFuture flatMap { case (successfulResults, failedResults) => + select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection => + val tagsFuture = tagsService.all + val parsedReports = selection.result + for{ + tagStatistics <- statisticsForTags(parsedReports, tagsFuture) + } yield Ok(views.html.statistics.basic( + tagStatistics = tagStatistics, + projectsWithSelection = selection.projectsWithSelection, + parsedReports = parsedReports + )) + } + } + } + + def statisticsForTags(parsedReports: DependencyCheckReportsParser.Result, tagsFuture: Future[Seq[(Int, LibraryTag)]]): Future[Seq[Statistics.TagStatistics]] = { + val librariesFuture = librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet) + val libraryTagAssignmentsFuture = librariesFuture.flatMap{libraries => libraryTagAssignmentsService.forLibraries(libraries.values.map(_._1).toSet)} + val tagsToLibrariesFuture = libraryTagAssignmentsService.tagsToLibraries(libraryTagAssignmentsFuture) + val librariesToDependencies = parsedReports.groupedDependenciesByPlainLibraryIdentifier + for{ + librariesById <- librariesFuture.map(_.values.toMap) + tagsToLibraries <- tagsToLibrariesFuture + tags <- tagsFuture + } yield tags.flatMap{case tagRecord @ (tagId, tag) => + val libraryAssignments = tagsToLibraries(tagId) + val tagLibraries = libraryAssignments.map(a => a.libraryId -> librariesById(a.libraryId)) + val tagDependencies: Set[GroupedDependency] = tagLibraries.flatMap{case (_, lib) => librariesToDependencies(lib.plainLibraryIdentifier)} + // TODO: vulnerabilities in the past + if(tagLibraries.isEmpty) None + else Some(TagStatistics(tagRecord = tagRecord, stats = LibDepStatistics(libraries = tagLibraries, dependencies = tagDependencies))) + } + } + + def vulnerabilities(projectOption: Option[String], tagIdOption: Option[Int]) = ReadAction.async {implicit req => + val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) + resultsFuture flatMap { case (successfulResults, failedResults) => + select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection => + val parsedReports = selection.result + for{ + libraries <- librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet) + tagOption <- tagIdOption.fold[Future[Option[(Int, LibraryTag)]]](Future.successful(None))(tagId => tagsService.getById(tagId).map(Some(_))) + statistics <- tagOption.fold(Future.successful(LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.values.toSet))){ tag => + statisticsForTags(parsedReports, Future.successful(Seq(tag))).map{ + case Seq(TagStatistics(_, stats)) => stats // statisticsForTags is designed for multiple tags, but we have just one… + case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set()) // We don't want to crash when no dependencies are there… + } + } + } yield Ok(views.html.statistics.vulnerabilities( + projectsWithSelection = selection.projectsWithSelection, + tagOption = tagOption, + statistics = statistics + )) + } + } + } + + def vulnerability(name: String, projectOption: Option[String]) = ReadAction.async { implicit req => + val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) + resultsFuture flatMap { case (successfulResults, failedResults) => + select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection => + val relevantReports = selection.result + val vulns = relevantReports.vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(vuln => (vuln, dep))).groupBy(_._1.name).mapValues{case vulnsWithDeps => + val (vulnSeq, depSeq) = vulnsWithDeps.unzip + val Seq(vuln) = vulnSeq.toSet.toSeq // Will fail when there are more different descriptions for one vulnerability… + vuln -> depSeq.toSet + }// .map(identity) // The .map(identity) materializes lazily mapped Map (because .mapValues is lazy). I am, however, unsure if this is a good idea. Probably not. + vulns.get(name).fold(Future.successful(Ok(views.html.statistics.vulnerabilityNotFound( + name = name, + projectsWithSelection = selection.projectsWithSelection + )))){ case (vuln, vulnerableDependencies) => + for( + plainLibs <- librariesService.byPlainLibraryIdentifiers(vulnerableDependencies.flatMap(_.plainLibraryIdentifiers)).map(_.keySet) + ) yield Ok(views.html.statistics.vulnerability( + vulnerability = vuln, + affectedProjects = vulnerableDependencies.flatMap(dep => dep.projects.map(proj => (proj, dep))).groupBy(_._1).mapValues(_.map(_._2)), + vulnerableDependencies = vulnerableDependencies, + affectedLibraries = plainLibs, + projectsWithSelection = selection.projectsWithSelection + )) + } + + } + } + } + + def vulnerableLibraries(project: Option[String]) = ReadAction.async { implicit req => + val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) + resultsFuture flatMap { case (successfulResults, failedResults) => + select(successfulResults, project).fold(Future.successful(notFound())){selection => + val reports = selection.result + Future.successful(Ok(views.html.statistics.vulnerableLibraries( + projectsWithSelection = selection.projectsWithSelection, + vulnerableDependencies = reports.vulnerableDependencies, + allDependenciesCount = reports.groupedDependencies.size + ))) + } + } + } + + def allLibraries(project: Option[String]) = ReadAction.async { implicit req => + val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) + resultsFuture flatMap { case (successfulResults, failedResults) => + select(successfulResults, project).fold(Future.successful(notFound())){selection => + Future.successful(Ok(views.html.statistics.allLibraries( + projectsWithSelection = selection.projectsWithSelection, + allDependencies = selection.result.groupedDependencies + ))) + } + } + } + + def allGavs(project: Option[String]) = ReadAction.async { implicit req => + val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) + resultsFuture flatMap { case (successfulResults, failedResults) => + select(successfulResults, project).fold(Future.successful(notFound())){selection => + Future.successful(Ok(Txt( + selection.result.groupedDependencies.flatMap(_.mavenIdentifiers).toSet.toIndexedSeq.sortBy((id: Identifier) => (id.identifierType, id.name)).map(id => id.name.split(':') match { + case Array(g, a, v) => + s""""${id.identifierType}", "$g", "$a", "$v", "${id.url}" """ + }).mkString("\n") + ))) + } + } + } + + +} diff --git a/app/controllers/package.scala b/app/controllers/package.scala new file mode 100644 index 0000000..0b76cae --- /dev/null +++ b/app/controllers/package.scala @@ -0,0 +1,37 @@ +import com.mohiva.play.silhouette.api.Environment +import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator +import models.{User, SnoozeInfo} + +/** + * Created by user on 7/15/15. + */ +package object controllers { + + // Imports for all templates. Those could be added directly to the template files, but IntelliJ IDEA does not like it. + type Dependency = com.ysoft.odc.Dependency + type Build = com.ysoft.odc.Build + type GroupedDependency = com.ysoft.odc.GroupedDependency + type Vulnerability = com.ysoft.odc.Vulnerability + type Identifier = com.ysoft.odc.Identifier + type DateTime = org.joda.time.DateTime + type SnoozesInfo = Map[String, SnoozeInfo] + type AuthEnv = Environment[User, CookieAuthenticator] + + + val NormalUrlPattern = """^(http(s?)|ftp(s?))://.*""".r + + val TooGenericDomains = Set("sourceforge.net", "github.com", "github.io") + + +/* def friendlyProjectName(unfriendlyName: String) = { + val (baseName, theRest) = unfriendlyName.span(_ != '/') + //theRest.drop(1) + val removeLeadingMess = RestMessBeginRegexp.replaceAllIn(_: String, "") + val removeTrailingMess = RestMessEndRegexp.replaceAllIn(_: String, "") + val removeMess = removeLeadingMess andThen removeTrailingMess + val subProjectOption = Some(removeMess(theRest)).filter(_ != "") + subProjectOption.fold(baseName)(baseName+"/"+_) + }*/ + def friendlyProjectName(reportInfo: ReportInfo) = reportInfo.subprojectNameOption.fold(reportInfo.projectName)(reportInfo.projectName+": "+_) + +} diff --git a/app/controllers/warnings.scala b/app/controllers/warnings.scala new file mode 100644 index 0000000..d78657a --- /dev/null +++ b/app/controllers/warnings.scala @@ -0,0 +1,21 @@ +package controllers + +import controllers.WarningSeverity.WarningSeverity +import play.twirl.api.Html + +object WarningSeverity extends Enumeration { + type WarningSeverity = Value + // Order is important + val Info = Value("info") + val Warning = Value("warning") + val Error = Value("error") +} + +sealed abstract class Warning { + def html: Html + def id: String + def allowSnoozes = true + def severity: WarningSeverity +} + +final case class IdentifiedWarning(id: String, html: Html, severity: WarningSeverity) extends Warning \ No newline at end of file diff --git a/app/models/CookieAuthenticators.scala b/app/models/CookieAuthenticators.scala new file mode 100644 index 0000000..9bc59a1 --- /dev/null +++ b/app/models/CookieAuthenticators.scala @@ -0,0 +1,33 @@ +package models + +import scala.concurrent.duration._ + +import com.mohiva.play.silhouette.api.LoginInfo +import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator +import models.profile.MappedJdbcType +import models.jodaSupport._ +import models.profile.api._ +import org.joda.time.DateTime +import slick.lifted.{ProvenShape, Tag} + +import scala.concurrent.duration.FiniteDuration + + +class CookieAuthenticators(tag: Tag) extends Table[CookieAuthenticator](tag, "cookie_authenticators") { + + private implicit val FiniteDurationType = MappedJdbcType.base[FiniteDuration, Long](_.toSeconds, FiniteDuration.apply(_, SECONDS)) + + def id = column[String]("id") + def providerId = column[String]("provider_id") + def providerKey = column[String]("provider_key") + def lastUsedDateTime = column[DateTime]("last_used") + def expirationDateTime = column[DateTime]("expiration") + def idleTimeout = column[FiniteDuration]("idle_timeout").? + def cookieMaxAge = column[FiniteDuration]("cookie_max_age").? + def fingerprint = column[String]("fingerprint").? + + def loginInfo = (providerId, providerKey) <> (LoginInfo.tupled, LoginInfo.unapply) + + override def * : ProvenShape[CookieAuthenticator] = (id, loginInfo, lastUsedDateTime, expirationDateTime, idleTimeout, cookieMaxAge, fingerprint) <> ((CookieAuthenticator.apply _).tupled, CookieAuthenticator.unapply) + +} diff --git a/app/models/Library.scala b/app/models/Library.scala new file mode 100644 index 0000000..d0d9409 --- /dev/null +++ b/app/models/Library.scala @@ -0,0 +1,52 @@ +package models + +import models.profile.MappedJdbcType +import models.profile.api._ +import slick.lifted.Tag + +abstract sealed class LibraryType(val name: String){ + override final def toString: String = name +} +object LibraryType{ + case object Maven extends LibraryType("maven") + case object DotNet extends LibraryType("dotnet") + val All = Set(Maven, DotNet) + val ByName = All.map(x => x.name -> x).toMap + implicit val libraryTypeMapper = MappedJdbcType.base[LibraryType, String](_.name, LibraryType.ByName) +} + +final case class Library(plainLibraryIdentifier: PlainLibraryIdentifier, classified: Boolean) + +final case class PlainLibraryIdentifier(libraryType: LibraryType, libraryIdentifier: String){ + override def toString: String = s"$libraryType:$libraryIdentifier" +} + +object PlainLibraryIdentifier extends ((LibraryType, String) => PlainLibraryIdentifier) { + def fromString(id: String) = { + val (libraryType, libraryNameWithColon) = id.span(_ != ':') + if(libraryNameWithColon(0) != ':'){ + sys.error("Expected colon") + } + val libraryName = libraryNameWithColon.drop(1) + PlainLibraryIdentifier( + libraryType = LibraryType.ByName(libraryType), + libraryIdentifier = libraryName + ) + } +} + +class Libraries(tag: Tag) extends Table[(Int, Library)](tag, "library") { + import LibraryType.libraryTypeMapper + def id = column[Int]("id", O.PrimaryKey) + def libraryType = column[LibraryType]("library_type") + def libraryIdentifier = column[String]("identifier") + def classified = column[Boolean]("classified") + + def plainLibraryIdentifierUnmapped = (libraryType, libraryIdentifier) + def plainLibraryIdentifier = plainLibraryIdentifierUnmapped <> (PlainLibraryIdentifier.tupled, PlainLibraryIdentifier.unapply) + + def base = (plainLibraryIdentifier, classified) <> (Library.tupled, Library.unapply) + def * = (id, base) + +} + diff --git a/app/models/LibraryTag.scala b/app/models/LibraryTag.scala new file mode 100644 index 0000000..6dc4df9 --- /dev/null +++ b/app/models/LibraryTag.scala @@ -0,0 +1,16 @@ +package models + +import models.profile.api._ +import slick.lifted.Tag + +final case class LibraryTag (name: String, note: Option[String], warningOrder: Option[Int]) + +class LibraryTags(tag: Tag) extends Table[(Int, LibraryTag)](tag, "library_tag") { + def id = column[Int]("id", O.PrimaryKey) + def name = column[String]("name") + def note = column[Option[String]]("note") + def warningOrder = column[Option[Int]]("warning_order") + + def base = (name, note, warningOrder) <> (LibraryTag.tupled, LibraryTag.unapply) + def * = (id, base) +} \ No newline at end of file diff --git a/app/models/LibraryTagAssignment.scala b/app/models/LibraryTagAssignment.scala new file mode 100644 index 0000000..8e6023d --- /dev/null +++ b/app/models/LibraryTagAssignment.scala @@ -0,0 +1,19 @@ +package models + +import models.profile.api._ +import slick.lifted.Tag + +final case class LibraryTagPair(libraryId: Int, tagId: Int) +final case class LibraryTagAssignment(libraryTagPair: LibraryTagPair, contextDependent: Boolean){ + def libraryId = libraryTagPair.libraryId + def tagId = libraryTagPair.tagId +} + +class LibraryTagAssignments(tag: Tag) extends Table[LibraryTagAssignment](tag, "library_to_library_tag") { + def libraryId = column[Int]("library_id") + def libraryTagId = column[Int]("library_tag_id") + def contextDependent = column[Boolean]("context_dependent") + + def libraryTagPair = (libraryId, libraryTagId) <> (LibraryTagPair.tupled, LibraryTagPair.unapply) + def * = (libraryTagPair, contextDependent) <> (LibraryTagAssignment.tupled, LibraryTagAssignment.unapply) +} \ No newline at end of file diff --git a/app/models/Snooze.scala b/app/models/Snooze.scala new file mode 100644 index 0000000..3e4ab3c --- /dev/null +++ b/app/models/Snooze.scala @@ -0,0 +1,44 @@ +package models + +import models.jodaSupport._ +import models.profile.api._ +import org.joda.time.LocalDate +import play.api.data.Form +import slick.lifted.Tag + +case class Snooze(until: LocalDate, snoozedObjectId: String, reason: String) + +case class ObjectSnooze(until: LocalDate, reason: String){ + def toSnooze(objectId: String) = Snooze(until, objectId, reason) +} + +class Snoozes(tag: Tag) extends Table[(Int, Snooze)](tag, "snooze") { + def id = column[Int]("id", O.PrimaryKey) + def until = column[LocalDate]("until") + def snoozedObjectId = column[String]("snoozed_object_identifier") + def reason = column[String]("reason") + def base = (until, snoozedObjectId, reason) <> (Snooze.tupled, Snooze.unapply) + def * = (id, base) +} + +case class SnoozeInfo(form: Form[ObjectSnooze], snoozes: Seq[(Int, Snooze)]){ + def shouldCollapse(default: Boolean) = { + shouldExpandForm match { + case true => false + case false => + isSnoozed match { + case true => true + case false => default + } + } + } + + def isSnoozed = snoozes.nonEmpty + + def shouldExpandForm = form.hasErrors || form.hasGlobalErrors + + def adjustForm(f: Form[ObjectSnooze] => Form[ObjectSnooze]): SnoozeInfo = copy(form = f(form)) + + def adjustSnoozes(f: Seq[(Int, Snooze)] => Seq[(Int, Snooze)]): SnoozeInfo = copy(snoozes = f(snoozes)) + +} \ No newline at end of file diff --git a/app/models/User.scala b/app/models/User.scala new file mode 100644 index 0000000..07750ca --- /dev/null +++ b/app/models/User.scala @@ -0,0 +1,5 @@ +package models + +import com.mohiva.play.silhouette.api.Identity + +case class User(username: String) extends Identity \ No newline at end of file diff --git a/app/models/odc/CpeEntry.scala b/app/models/odc/CpeEntry.scala new file mode 100644 index 0000000..3fee69f --- /dev/null +++ b/app/models/odc/CpeEntry.scala @@ -0,0 +1,20 @@ +package models.odc + +import models.odc.profile.api._ +import slick.lifted.Tag + +final case class CpeEntry(cpe: String, vendor: String, product: String) + +class CpeEntries(tag: Tag) extends Table[(Int, CpeEntry)](tag, "cpeEntry") { + + def id = column[Int]("id", O.PrimaryKey) + + def cpe = column[String]("cpe") + def vendor = column[String]("vendor") + def product = column[String]("product") + + def base = (cpe, vendor, product) <> (CpeEntry.tupled, CpeEntry.unapply) + + def * = (id, base) + +} diff --git a/app/models/odc/OdcProperty.scala b/app/models/odc/OdcProperty.scala new file mode 100644 index 0000000..67f1390 --- /dev/null +++ b/app/models/odc/OdcProperty.scala @@ -0,0 +1,11 @@ +package models.odc +import models.odc.profile.api._ + +final case class OdcProperty (id: String, value: String) + +final class OdcProperties(tag: Tag) extends Table[OdcProperty](tag, "properties"){ + def id = column[String]("id") + def value = column[String]("value") + + def * = (id, value) <> (OdcProperty.tupled, OdcProperty.unapply) +} \ No newline at end of file diff --git a/app/models/odc/References.scala b/app/models/odc/References.scala new file mode 100644 index 0000000..214dee8 --- /dev/null +++ b/app/models/odc/References.scala @@ -0,0 +1,17 @@ +package models.odc + +import com.ysoft.odc.Reference +import models.odc +import models.odc.profile.MappedJdbcType +import models.odc.profile.api._ +import slick.lifted.Tag + +class References (tag: Tag) extends Table[(Int, Reference)](tag, "reference") { + def cveId = column[Int]("cveid") + def name = column[String]("name") + def url = column[String]("url") + def source = column[String]("source") + + def base = (source, url, name) <> (Reference.tupled, Reference.unapply) + def * = (cveId, base) +} diff --git a/app/models/odc/SoftwareVulnerability.scala b/app/models/odc/SoftwareVulnerability.scala new file mode 100644 index 0000000..9c0aefc --- /dev/null +++ b/app/models/odc/SoftwareVulnerability.scala @@ -0,0 +1,39 @@ +package models.odc + +import models.odc.profile.api._ +import models.odc.profile.jdbcTypeFor +import slick.ast.TypedType +import models.odc.profile.MappedJdbcType +import slick.jdbc.JdbcType + +import scala.reflect.ClassTag + +// TODO: consider renaming to CpeEntryVulnerability or something like that +final case class SoftwareVulnerability (vulnerabilityId: Int, cpeEntryId: Int, includesAllPreviousVersionsRaw: Option[String]){ + def includesAllPreviousVersions: Boolean = includesAllPreviousVersionsRaw match { + case Some("1") => true + case None => false + } +} + +/*private class OdcBooleanType(implicit t: JdbcType[Option[String]]) extends MappedJdbcType[Boolean, Option[String]] { + override def map(t: Boolean): Option[String] = t match { + case true => Some("1") + case false => None + } + + override def comap(u: Option[String]): Boolean = u match { + case Some("1") => true + case None => false + } + +}*/ + +class SoftwareVulnerabilities(tag: Tag) extends Table[SoftwareVulnerability](tag, "software") { + def vulnerabilityId = column[Int]("cveid") + def cpeEntryId = column[Int]("cpeEntryId") + //private val bt = new OdcBooleanType()(jdbcTypeFor(implicitly[BaseColumnType[String]].optionType).asInstanceOf[JdbcType[Option[String]]]) + //MappedJdbcType.base[Boolean, Option[String]](???, ???)(implicitly[ClassTag[Boolean]], ) + def includesAllPreviousVersionsRaw = column[String]("previousVersion").? + def * = (vulnerabilityId, cpeEntryId, includesAllPreviousVersionsRaw) <> (SoftwareVulnerability.tupled, SoftwareVulnerability.unapply) +} \ No newline at end of file diff --git a/app/models/odc/Vulnerability.scala b/app/models/odc/Vulnerability.scala new file mode 100644 index 0000000..50a3873 --- /dev/null +++ b/app/models/odc/Vulnerability.scala @@ -0,0 +1,26 @@ +package models.odc + +import com.ysoft.odc.{CvssRating, CWE} +import models.odc.profile.api._ +import slick.lifted.Tag + +case class Vulnerability (cve: String, description: String, cweOption: Option[CWE], cvss: CvssRating) +class Vulnerabilities(tag: Tag) extends Table[(Int, Vulnerability)](tag, "vulnerability") { + def id = column[Int]("id") + def cve = column[String]("cve") + def description = column[String]("description") + def cweOption = column[String]("cwe").? + def cvssScore = column[Double]("cvssScore").? + def authentication = column[String]("cvssAuthentication").? + def availabilityImpact = column[String]("cvssAvailabilityImpact").? + def accessVector = column[String]("cvssAccessVector").? + def integrityImpact = column[String]("cvssIntegrityImpact").? + def cvssAccessComplexity = column[String]("cvssAccessComplexity").? + def cvssConfidentialityImpact = column[String]("cvssConfidentialityImpact").? + + def cvssRating = (cvssScore, authentication, availabilityImpact, accessVector, integrityImpact, cvssAccessComplexity, cvssConfidentialityImpact) <> (CvssRating.tupled, CvssRating.unapply) + def cweOptionMapped = cweOption <> ((_: Option[String]).map(CWE.apply), (_: Option[CWE]).map(CWE.unapply)) + def base = (cve, description, cweOptionMapped, cvssRating) <> (Vulnerability.tupled, Vulnerability.unapply) + + def * = (id, base) +} \ No newline at end of file diff --git a/app/models/odc/package.scala b/app/models/odc/package.scala new file mode 100644 index 0000000..c02f90e --- /dev/null +++ b/app/models/odc/package.scala @@ -0,0 +1,17 @@ +package models + +import slick.lifted.TableQuery + +package object odc { + + val profile = slick.driver.MySQLDriver + + object tables { + val cpeEntries = TableQuery[CpeEntries] + val softwareVulnerabilities = TableQuery[SoftwareVulnerabilities] + val vulnerabilities = TableQuery[Vulnerabilities] + val references = TableQuery[References] + val properties = TableQuery[OdcProperties] + } + +} diff --git a/app/models/package.scala b/app/models/package.scala new file mode 100644 index 0000000..94d2e67 --- /dev/null +++ b/app/models/package.scala @@ -0,0 +1,20 @@ +import slick.lifted.TableQuery + +/** + * Created by user on 8/12/15. + */ +package object models { + + val profile = slick.driver.PostgresDriver + + val jodaSupport = com.github.tototoshi.slick.PostgresJodaSupport + + object tables { + val libraries = TableQuery[Libraries] + val libraryTagAssignments = TableQuery[LibraryTagAssignments] + val tags = TableQuery[LibraryTags] + val snoozesTable = TableQuery[Snoozes] + val authTokens = TableQuery[CookieAuthenticators] + } + +} diff --git a/app/modules/ConfigModule.scala b/app/modules/ConfigModule.scala new file mode 100644 index 0000000..ac7e99f --- /dev/null +++ b/app/modules/ConfigModule.scala @@ -0,0 +1,99 @@ +package modules + +import java.io._ +import java.net.URLEncoder +import java.nio.file.{Files, Path, Paths} + +import akka.util.ClassLoaderObjectInputStream +import com.ysoft.odc._ +import controllers.MissingGavExclusions +import play.api.cache.CacheApi +import play.api.inject.{Binding, Module} +import play.api.{Configuration, Environment, Logger} + +import scala.concurrent.duration.Duration +import scala.reflect.ClassTag +import scala.util.{Failure, Success, Try} + +/** + * This class is rather a temporary hack and should be replaced by something better. + * + * Issues: + * * Thread safety + * * fsync: https://stackoverflow.com/questions/4072878/i-o-concept-flush-vs-sync + * * probably not removing files that are not used for a long time + * @param path + */ +class FileCacheApi(path: Path) extends CacheApi{ + private def cacheFile(name: String) = path.resolve("X-"+URLEncoder.encode(name, "utf-8")) + override def remove(key: String): Unit = Files.deleteIfExists(cacheFile(key)) + + private def serialize(value: Any, duration: Duration) = { + val out = new ByteArrayOutputStream() + import com.github.nscala_time.time.Imports._ + new ObjectOutputStream(out).writeObject((value, if(duration.isFinite()) Some(DateTime.now.plus(duration.toMillis)) else None)) + out.toByteArray + } + + private def unserialize[T](data: Array[Byte]): Try[T] = { + val in = new ByteArrayInputStream(data) + import com.github.nscala_time.time.Imports._ + try{ + new ClassLoaderObjectInputStream(this.getClass.getClassLoader, in).readObject() match { + case (value, None) => Success(value.asInstanceOf[T]) + case (value, Some(exp: DateTime)) if exp < DateTime.now => Success(value.asInstanceOf[T]) + case _ => Failure(new RuntimeException("cache expired")) + } + }catch{ + case e: ObjectStreamException => Failure(e) + } + } + + override def set(key: String, value: Any, expiration: Duration): Unit = { + Files.write(cacheFile(key), serialize(value, expiration)) + } + + override def get[T: ClassTag](key: String): Option[T] = { + val f = cacheFile(key) + if(Files.exists(f)){ + val res = unserialize[T](Files.readAllBytes(f)) + res match { + case Failure(e) => + Logger.warn("not using cache for following key, removing that: "+key, e) + remove(key) + case Success(_) => // nothing to do + } + res.toOption + }else{ + None + } + } + + override def getOrElse[A: ClassTag](key: String, expiration: Duration)(orElse: => A): A = get(key).getOrElse{ + val v = orElse + set(key, v, expiration) + v + } + +} + + +class ConfigModule extends Module { + + override def bindings(environment: Environment, configuration: Configuration): Seq[Binding[_]] = Seq( + bind[String].qualifiedWith("bamboo-server-url").toInstance(configuration.getString("yssdc.bamboo.url").getOrElse(sys.error("Key yssdc.bamboo.url is not set"))), + configuration.getString("yssdc.reports.provider") match{ + case Some("bamboo") => bind[Downloader].to[BambooDownloader] + // not ready yet: case Some("files") => bind[Downloader].to[LocalFilesDownloader] + case other => sys.error(s"unknown provider: $other") + }, + bind[MissingGavExclusions].qualifiedWith("missing-GAV-exclusions").toInstance(MissingGavExclusions( + configuration.getStringSeq("yssdc.exclusions.missingGAV.bySha1").getOrElse(Seq()).toSet.map(Exclusion)) + ) + ) ++ + configuration.getString("play.cache.path").map(cachePath => bind[CacheApi].toInstance(new FileCacheApi(Paths.get(cachePath)))) ++ + configuration.getString("yssdc.reports.bamboo.sessionId").map{s => bind[BambooAuthentication].toInstance(new SessionIdBambooAuthentication(s))} ++ + configuration.getString("yssdc.reports.bamboo.user").map{u => bind[BambooAuthentication].toInstance(new CredentialsBambooAuthentication(u, configuration.getString("yssdc.reports.bamboo.password").get))} ++ + configuration.getString("yssdc.reports.path").map{s => bind[String].qualifiedWith("reports-path").toInstance(s)} + +} diff --git a/app/modules/SilhouetteModule.scala b/app/modules/SilhouetteModule.scala new file mode 100644 index 0000000..aa6c686 --- /dev/null +++ b/app/modules/SilhouetteModule.scala @@ -0,0 +1,78 @@ +package modules + +import com.google.inject.{AbstractModule, Provides} +import com.mohiva.play.silhouette.api.repositories.AuthInfoRepository +import com.mohiva.play.silhouette.api.util._ +import com.mohiva.play.silhouette.api.{Environment, EventBus} +import com.mohiva.play.silhouette.api.services.AuthenticatorService +import com.mohiva.play.silhouette.impl.authenticators.{CookieAuthenticatorService, CookieAuthenticatorSettings, CookieAuthenticator} +import com.mohiva.play.silhouette.impl.daos.DelegableAuthInfoDAO +import com.mohiva.play.silhouette.impl.providers.{CredentialsProvider, SocialProviderRegistry} +import com.mohiva.play.silhouette.impl.repositories.DelegableAuthInfoRepository +import com.mohiva.play.silhouette.impl.util.{BCryptPasswordHasher, SecureRandomIDGenerator, DefaultFingerprintGenerator} +import models.User +import net.codingwell.scalaguice.ScalaModule +import play.api.libs.ws.WSClient +import play.api.{Application, Configuration} +import services._ +import net.ceedubs.ficus.Ficus._ +import net.ceedubs.ficus.readers.ArbitraryTypeReader._ +import play.api.libs.concurrent.Execution.Implicits._ + +class SilhouetteModule extends AbstractModule with ScalaModule{ + + + override def configure(): Unit = { + bind[FingerprintGenerator].toInstance(new DefaultFingerprintGenerator(false)) + bind[IDGenerator].toInstance(new SecureRandomIDGenerator()) + bind[Clock].toInstance(Clock()) + } + + @Provides + def provideAuthInfoRepository(passwordInfoDAO: DelegableAuthInfoDAO[PasswordInfo]): AuthInfoRepository = { + new DelegableAuthInfoRepository(passwordInfoDAO) + } + + @Provides + def provideAuthenticatorService( + fingerprintGenerator: FingerprintGenerator, + idGenerator: IDGenerator, + configuration: Configuration, + clock: Clock, + tokenService: TokenService + ): AuthenticatorService[CookieAuthenticator] = { + val config = configuration.underlying.as[CookieAuthenticatorSettings]("silhouette.authenticator") + new CookieAuthenticatorService( + config, + Some(tokenService), + fingerprintGenerator, + idGenerator, + clock + ) + } + + @Provides + def provideCredentialsVerificationService(configuration: Configuration, app: Application)(implicit wSClient: WSClient): CredentialsVerificationService = { + configuration.underlying.as[String]("silhouette.credentialsVerificationService.type") match { + case "allow-all" => new AllowAllCredentialsVerificationService(app) + case "external" => new ExternalCredentialsVerificationService(configuration.underlying.as[String]("silhouette.credentialsVerificationService.url")) + } + } + + @Provides + def provide(userService: UserService): DelegableAuthInfoDAO[PasswordInfo] = userService + + @Provides + def provideEnvironment( + userService: UserService, + authenticatorService: AuthenticatorService[CookieAuthenticator], + eventBus: EventBus): Environment[User, CookieAuthenticator] = { + Environment[User, CookieAuthenticator]( + userService, + authenticatorService, + Seq(), + eventBus + ) + } + +} diff --git a/app/services/AllowAllCredentialsVerificationService.scala b/app/services/AllowAllCredentialsVerificationService.scala new file mode 100644 index 0000000..d9f6f92 --- /dev/null +++ b/app/services/AllowAllCredentialsVerificationService.scala @@ -0,0 +1,13 @@ +package services + +import scala.concurrent.Future + +class AllowAllCredentialsVerificationService(app: play.api.Application) extends CredentialsVerificationService{ + if(app.mode != play.api.Mode.Dev){ + // safety check: + sys.error("allow-all can be used in dev mode only") + } + + override def verifyCredentials(username: String, password: String): Future[Boolean] = Future.successful(true) + +} diff --git a/app/services/CredentialsVerificationService.scala b/app/services/CredentialsVerificationService.scala new file mode 100644 index 0000000..2c1292c --- /dev/null +++ b/app/services/CredentialsVerificationService.scala @@ -0,0 +1,7 @@ +package services + +import scala.concurrent.Future + +trait CredentialsVerificationService { + def verifyCredentials(username: String, password: String): Future[Boolean] +} diff --git a/app/services/ExternalCredentialsVerificationService.scala b/app/services/ExternalCredentialsVerificationService.scala new file mode 100644 index 0000000..96162b5 --- /dev/null +++ b/app/services/ExternalCredentialsVerificationService.scala @@ -0,0 +1,17 @@ +package services + +import play.api.libs.json.Json +import play.api.libs.ws.{WS, WSClient} + +import scala.concurrent.{Future, ExecutionContext} + +class ExternalCredentialsVerificationService(url: String)(implicit executionContext: ExecutionContext, wSClient: WSClient) extends CredentialsVerificationService{ + override def verifyCredentials(username: String, password: String): Future[Boolean] = { + WS.clientUrl(url).post(Json.toJson(Map("username" -> username, "password" -> password))).map{ response => + response.body match { + case "OK" => true + case "bad" => false + } + } + } +} diff --git a/app/services/LibrariesService.scala b/app/services/LibrariesService.scala new file mode 100644 index 0000000..f8c9a53 --- /dev/null +++ b/app/services/LibrariesService.scala @@ -0,0 +1,78 @@ +package services + +import com.google.inject.Inject +import models._ +import models.tables._ +import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} + +import scala.concurrent.{ExecutionContext, Future} + +class LibrariesService @Inject() (protected val dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends HasDatabaseConfigProvider[models.profile.type]{ + import dbConfig.driver.api._ + + def all: Future[Seq[(Int, Library)]] = db.run(libraries.result) + + def allBase: Future[Seq[Library]] = db.run(libraries.map(_.base).result) + + // TODO: unify or differentiate librariesForTags and byTags + def librariesForTags(tagIds: Iterable[Int]): Future[Seq[(Int, (Int, Library))]] = db.run( + libraryTagAssignments + .join(libraries).on { case (a, l) => a.libraryId === l.id } + .filter { case (a, l) => a.libraryTagId inSet tagIds } + .map { case (a, l) => a.libraryTagId -> l } + .result + ) + + def byTags(libraryIds: Set[Int]) = { + (db.run( + libraryTagAssignments + .filter(_.libraryId inSet libraryIds) + .result + ): Future[Seq[LibraryTagAssignment]]).map(_.groupBy(_.libraryId).mapValues(_.toSet).map(identity)) + } + + + def setClassified(libraryId: Int, classified: Boolean): Future[_] = db.run(libraries.filter(_.id === libraryId).map(_.classified).update(classified)) + + def touched(tagIds: Iterable[Int]): Future[Seq[(Int, Library)]] = db.run(libraries.filter(l => l.classified || l.id.inSet(tagIds) ).result) + + def unclassified: Future[Seq[(Int, Library)]] = db.run(libraries.filter(!_.classified).sortBy(l => l.plainLibraryIdentifierUnmapped).result) + + def insert(lib: Library) = db.run(libraries.map(_.base).returning(libraries.map(_.id)) += lib) + + def insertMany(newItems: Iterable[Library]): Future[_] = db.run(libraries.map(_.base) ++= newItems) + + def filtered(requiredClassification: Option[Boolean], requiredTagsOption: Option[Set[Int]]): Query[Libraries, (Int, Library), Seq] = { + libraries + .joinLeft(libraryTagAssignments).on { case (l, a) => l.id === a.libraryId } + .filter { case (l, a) => requiredClassification.map(l.classified === _).getOrElse(l.classified === l.classified) } // classification matches + .filter { case (l, a) => + requiredTagsOption.fold( + // actually a.isEmpty; The t.isEmpty should work, but there is a bug – it uses (… = 1) on a null value, which has a different semantics in SQL. Related to https://github.com/slick/slick/issues/1156 . + // So, I created following workaround: + a.fold(true.asColumnOf[Boolean])(_ => false) // Filter only libraries with no tags (i.e. LEFT JOIN added NULLs (they corresponsd to None value)) + )(requiredTagsSet => + if (requiredTagsSet.isEmpty) true.asColumnOf[Boolean] // If we don't filter any by tag, we should allow all + else a.map(_.libraryTagId inSet requiredTagsSet).getOrElse(false.asColumnOf[Boolean]) // Filter tags + ) + } + .groupBy { case (l, a) => l } // a library with multiple tags should be present only once + .map { case (l, q) => (l, q.size) } // we are not interested in the tags, but only in their count + .filter { case (l, c) => requiredTagsOption.fold( true.asColumnOf[Boolean] )(requiredTagsSet => c >= requiredTagsSet.size ) } // filter libraries with all the tags we are looking for + .map { case (l, c) => l } // all is filtered, so we need libraries only + .sortBy { l => l.plainLibraryIdentifierUnmapped } + } + + def byPlainLibraryIdentifiers(plainLibraryIdentifiers: Set[PlainLibraryIdentifier]): Future[Map[PlainLibraryIdentifier, (Int, Library)]] = { + val groupedIdentifiers = plainLibraryIdentifiers.groupBy(_.libraryType).mapValues(_.map(_.libraryIdentifier)).map(identity) + val resFuture: Future[Seq[(Int, Library)]] = db.run(libraries.filter{l => + val conditions = for((libraryType, identifiers) <- groupedIdentifiers) yield l.libraryType === libraryType && l.libraryIdentifier.inSet(identifiers) + conditions.foldLeft(false.asColumnOf[Boolean])(_ || _) + }.result) + resFuture.map( + //_.toSet.groupBy(_._2.plainLibraryIdentifier) + _.map(x => x._2.plainLibraryIdentifier -> x).toMap + ) + } + +} diff --git a/app/services/LibraryTagAssignmentsService.scala b/app/services/LibraryTagAssignmentsService.scala new file mode 100644 index 0000000..d25f601 --- /dev/null +++ b/app/services/LibraryTagAssignmentsService.scala @@ -0,0 +1,34 @@ +package services + +import com.google.inject.Inject +import models.tables._ +import models.{LibraryTagAssignment, LibraryTagPair} +import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} + +import scala.concurrent.{ExecutionContext, Future} +class LibraryTagAssignmentsService @Inject() (protected val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[models.profile.type]{ + import dbConfig.driver.api._ + + + def all = db.run(libraryTagAssignments.result): Future[Seq[LibraryTagAssignment]] + + def insert(item: LibraryTagAssignment) = db.run(libraryTagAssignments += item) + + def remove(libraryTagPair: LibraryTagPair) = db.run( + libraryTagAssignments + .filter(_.libraryTagId === libraryTagPair.tagId) + .filter(_.libraryId === libraryTagPair.libraryId) + .delete + ) + + def forLibraries(libraryIds: Set[Int]): Future[Seq[LibraryTagAssignment]] = db.run(libraryTagAssignments.filter(_.libraryId inSet libraryIds).result) + + def byLibrary(implicit executionContext: ExecutionContext) = all.map(_.groupBy(_.libraryId).withDefaultValue(Seq())) + + def tagsToLibraries(tagAssignmentsFuture: Future[Seq[LibraryTagAssignment]])(implicit executionContext: ExecutionContext): Future[Map[Int, Set[LibraryTagAssignment]]] = + tagAssignmentsFuture.map(x => tagsToLibraries(x)) + + def tagsToLibraries(tagAssignments: Seq[LibraryTagAssignment]): Map[Int, Set[LibraryTagAssignment]] = tagAssignments.groupBy(_.tagId).mapValues(_.toSet).map(identity).withDefaultValue(Set.empty) + + +} diff --git a/app/services/OdcService.scala b/app/services/OdcService.scala new file mode 100644 index 0000000..b8971bf --- /dev/null +++ b/app/services/OdcService.scala @@ -0,0 +1,124 @@ +package services + +import java.lang.{Boolean => JBoolean} +import java.util.{Map => JMap} + +import _root_.org.owasp.dependencycheck.data.nvdcve.CveDB +import _root_.org.owasp.dependencycheck.dependency.VulnerableSoftware +import _root_.org.owasp.dependencycheck.utils.{DependencyVersion, DependencyVersionUtil, Settings} +import com.github.nscala_time.time.Imports._ +import com.google.inject.Inject +import models.odc.OdcProperty +import models.odc.tables._ +import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} +import play.db.NamedDatabase + +import scala.concurrent.{ExecutionContext, Future} + +class OdcService @Inject()(@NamedDatabase("odc") protected val dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends HasDatabaseConfigProvider[models.odc.profile.type]{ + + import dbConfig.driver.api._ + + def getVulnerableSoftware(id: Int): Future[Seq[com.ysoft.odc.VulnerableSoftware]] = { + db.run(softwareVulnerabilities.joinLeft(cpeEntries).on((sv, ce) => sv.cpeEntryId === ce.id).filter{case (sv, ceo) => sv.vulnerabilityId === id}.result).map{rawRefs => + rawRefs.map{ + case (softVuln, Some((_, cpeEntry))) => com.ysoft.odc.VulnerableSoftware(allPreviousVersion = softVuln.includesAllPreviousVersions, name=cpeEntry.cpe) + } + } + } + + def getReferences(id: Int): Future[Seq[com.ysoft.odc.Reference]] = db.run(references.filter(_.cveId === id).map(_.base).result) + + def getVulnerabilityDetails(id: Int): Future[Option[com.ysoft.odc.Vulnerability]] = { + for { + bareVulnOption <- db.run(vulnerabilities.filter(_.id === id).map(_.base).result).map(_.headOption) + vulnerableSoftware <- getVulnerableSoftware(id) + references <- getReferences(id) + } yield bareVulnOption.map{bareVuln => + com.ysoft.odc.Vulnerability( + name = bareVuln.cve, + cweOption = bareVuln.cweOption, + cvss = bareVuln.cvss, + description = bareVuln.description, + vulnerableSoftware = vulnerableSoftware, + references = references + ) + } + } + + private def parseCpe(cpe: String) = { + val sw = new VulnerableSoftware() + sw.parseName(cpe) + sw + } + + def parseVersion(version: String): DependencyVersion = { + DependencyVersionUtil.parseVersion(version) + } + + /*def parseCpeVersion(cpe: String): DependencyVersion = { // strongly inspired by org.owasp.dependencycheck.data.nvdcve.CveDB.parseDependencyVersion(cpe: VulnerableSoftware): DependencyVersion + def StringOption(s: String) = Option(s).filterNot(_.isEmpty) + val sw = parseCpe(cpe) + StringOption(sw.getVersion) match { + case None ⇒ new DependencyVersion("-") + case Some(bareVersionString) ⇒ + DependencyVersionUtil.parseVersion( + StringOption(sw.getUpdate) match { + case None ⇒ bareVersionString + case Some(update) ⇒ s"$bareVersionString.$update" + } + ) + } + }*/ + + def findRelevantCpes(versionlessCpe: String, version: String) = { + println(s"versionlessCpe: $versionlessCpe") + val Seq("cpe", "/a", vendor, product, rest @ _*) = versionlessCpe.split(':').toSeq + val cpesFuture = db.run( + cpeEntries.filter(c => + c.vendor === vendor && c.product === product + ).result + ) + for(cpes <- cpesFuture){println(s"cpes: $cpes")} + val cpesMapFuture = cpesFuture.map(_.toMap) + val cpeIdsFuture = cpesFuture.map(_.map(_._1)) + val parsedVersion = parseVersion(version) + val res = for{ + cpeIds <- cpeIdsFuture + relevantVulnerabilities <- db.run( + softwareVulnerabilities.join(vulnerabilities).on( (sv, v) => sv.vulnerabilityId === v.id) + .filter{case (sv, v) => sv.cpeEntryId inSet cpeIds}.map{case (sv, v) ⇒ sv}.result + ).map(_.groupBy(_.vulnerabilityId).mapValues(_.toSet)) + cpesMap <- cpesMapFuture + //relevantVulnerabilities <- db.run(vulnerabilities.filter(_.id inSet relevantVulnerabilityIds).result) + } yield relevantVulnerabilities.filter{case (vulnId, sv) => Option(CveDbHelper.matchSofware( + vulnerableSoftware = sv.map(sv => cpesMap(sv.cpeEntryId).cpe -> sv.includesAllPreviousVersions).toMap, + vendor = vendor, + product = product, + identifiedVersion = parsedVersion + )).isDefined} + res.map(_.values.toSet.flatten) + } + + def loadUpdateProperties(): Future[Map[String, Long]] = db.run(properties.filter(_.id like "NVD CVE%").result).map(_.map{case OdcProperty(id, value) => (id, value.toLong)}.toMap) + + def loadLastDbUpdate(): Future[DateTime] = loadUpdateProperties().map(vals => new DateTime(vals.values.max)) // TODO: timezone (I don't care much, though) + +} + + +private[services] object CveDbHelper { + + + def matchSofware(vulnerableSoftware: Map[String, Boolean], vendor: String, product: String, identifiedVersion: DependencyVersion) = { + if(Settings.getInstance() == null){ + Settings.initialize()// Initiallize ODC environment on first use; Needed for each thread. + } + val cd = new CveDB() + import scala.collection.JavaConversions._ + val method = cd.getClass.getDeclaredMethod("getMatchingSoftware", classOf[JMap[String, JBoolean]], classOf[String], classOf[String], classOf[DependencyVersion]) + method.setAccessible(true) + method.invoke(cd, mapAsJavaMap(vulnerableSoftware).asInstanceOf[JMap[String, JBoolean]], vendor, product, identifiedVersion) + } +} + diff --git a/app/services/TagsService.scala b/app/services/TagsService.scala new file mode 100644 index 0000000..e64ecdf --- /dev/null +++ b/app/services/TagsService.scala @@ -0,0 +1,19 @@ +package services + +import com.google.inject.Inject +import models._ +import models.tables._ +import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} + +import scala.concurrent.{ExecutionContext, Future} + +class TagsService @Inject() (protected val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[models.profile.type]{ + import dbConfig.driver.api._ + + def all: Future[Seq[(Int, LibraryTag)]] = db.run(tags.result) + + def insertMany(newTags: Iterable[LibraryTag]): Future[_] = db.run(tags.map(_.base) ++= newTags) + + def getById(id: Int)(implicit executionContext: ExecutionContext): Future[(Int, LibraryTag)] = db.run(tags.filter(_.id === id).result).map(_.head) + +} diff --git a/app/services/TokenService.scala b/app/services/TokenService.scala new file mode 100644 index 0000000..180c3a2 --- /dev/null +++ b/app/services/TokenService.scala @@ -0,0 +1,36 @@ +package services + +import javax.inject.Inject + +import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator +import com.mohiva.play.silhouette.impl.daos.AuthenticatorDAO +import play.api.db.slick.{HasDatabaseConfigProvider, DatabaseConfigProvider} +import models.tables._ + +import scala.concurrent.{Future, ExecutionContext} + + +class TokenService @Inject() (protected val dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) + extends AuthenticatorDAO[CookieAuthenticator] + with HasDatabaseConfigProvider[models.profile.type]{ + import dbConfig.driver.api._ + + println(authTokens.schema.create.statements.toIndexedSeq) + + override def find(id: String): Future[Option[CookieAuthenticator]] = { + db.run(authTokens.filter(_.id === id).result).map{_.headOption} + } + + override def add(authenticator: CookieAuthenticator): Future[CookieAuthenticator] = { + db.run(authTokens += authenticator).map(_ => authenticator) + } + + override def remove(id: String): Future[Unit] = { + db.run(authTokens.filter(_.id === id).delete).map(_ => ()) + } + + override def update(authenticator: CookieAuthenticator): Future[CookieAuthenticator] = { + db.run(authTokens.filter(_.id === authenticator.id).update(authenticator)).map(_ => authenticator) + } + +} diff --git a/app/services/UserService.scala b/app/services/UserService.scala new file mode 100644 index 0000000..5fee8dd --- /dev/null +++ b/app/services/UserService.scala @@ -0,0 +1,29 @@ +package services + +import com.mohiva.play.silhouette.api.LoginInfo +import com.mohiva.play.silhouette.api.services.IdentityService +import com.mohiva.play.silhouette.api.util.PasswordInfo +import com.mohiva.play.silhouette.impl.daos.DelegableAuthInfoDAO +import models.User + +import scala.concurrent.Future + +class UserService extends DelegableAuthInfoDAO[PasswordInfo] with IdentityService[User] +{ + override def retrieve(loginInfo: LoginInfo): Future[Option[User]] = if(loginInfo.providerID == "credentials-verification") Future.successful(Some(User(loginInfo.providerKey))) else Future.successful(None) + + override def find(loginInfo: LoginInfo): Future[Option[PasswordInfo]] = { + println(s"loginInfo: $loginInfo") + + ??? + } + + override def update(loginInfo: LoginInfo, authInfo: PasswordInfo): Future[PasswordInfo] = ??? + + override def remove(loginInfo: LoginInfo): Future[Unit] = ??? + + override def save(loginInfo: LoginInfo, authInfo: PasswordInfo): Future[PasswordInfo] = ??? + + override def add(loginInfo: LoginInfo, authInfo: PasswordInfo): Future[PasswordInfo] = ??? + +} diff --git a/app/views/auth/signIn.scala.html b/app/views/auth/signIn.scala.html new file mode 100644 index 0000000..05812b0 --- /dev/null +++ b/app/views/auth/signIn.scala.html @@ -0,0 +1,12 @@ +@import helper._ +@(loginForm: Form[LoginRequest]/*, socialProviderRegistry: SocialProviderRegistry*/)(implicit requestHeader: DefaultRequest, messages: Messages) + +@main("Log in"){ + @form(routes.AuthController.authenticate()){ + @CSRF.formField + @inputText(loginForm("username")) + @inputPassword(loginForm("password")) + @* checkbox(loginForm("rememberMe")) *@ + + } +} \ No newline at end of file diff --git a/app/views/conditionalList.scala.html b/app/views/conditionalList.scala.html new file mode 100644 index 0000000..9516dbe --- /dev/null +++ b/app/views/conditionalList.scala.html @@ -0,0 +1,18 @@ +@(list: Traversable[_], name: String, id: String, collapse: Boolean = false, allowSnoozes: Boolean = true, versions: Map[String, Int])(content: => Html)(implicit rh: DefaultRequest, snoozes: SnoozesInfo, messages: Messages) +@if(list.nonEmpty){ + @defining(snoozes(id)){ case si => + @if(allowSnoozes) { + @snoozeButton(id, si, collapse) + } +

@if(si.isSnoozed){😴} @name (@{list.size})

+
+ @if(allowSnoozes) { + @snoozeForm(id, si, versions) + } + @content + @if(allowSnoozes) { + @snoozesList(id, si, versions) + } +
+ } +} \ No newline at end of file diff --git a/app/views/dependencies.scala.html b/app/views/dependencies.scala.html new file mode 100644 index 0000000..35a10f4 --- /dev/null +++ b/app/views/dependencies.scala.html @@ -0,0 +1,40 @@ +@( + selectedDependencies: Seq[(Int, Library)], + allTags: Seq[(Int, LibraryTag)], + dependencyTags: Map[Int, Set[LibraryTagAssignment]], + requiredClassification: Option[Boolean], + requiredTagSet: Set[Int], + noTag: Boolean, + tagsLink: Set[Int] => Call, + classificationLink: Option[Boolean] => Call, + noTagLink: Boolean => Call +)(implicit header: DefaultRequest) +@main(s"${requiredClassification match{case Some(true) => "Classified" case Some(false) => "Unclassified" case None => "All"}} dependencies (${selectedDependencies.size})") { +
+
+ @for((newClassification, name) <- Seq(None -> "All", Some(true) -> "Classified", Some(false) -> "Unclassified"); isCurrent = newClassification == requiredClassification){ + @name + } +
+
+ Required tags: + @if(!noTag) { + @for((tagId, tag) <- allTags.sortBy(_._2.name); enabled = requiredTagSet contains tagId) { + @tag.name + } + } +
+ @dependencyClassification( + prefix = "dependency", + dependencies = selectedDependencies, + allTags = allTags, + dependenciesTags = dependencyTags, + details = (_, _) => { + Html("") + } + ) +} \ No newline at end of file diff --git a/app/views/dependencyClassification.scala.html b/app/views/dependencyClassification.scala.html new file mode 100644 index 0000000..accf93c --- /dev/null +++ b/app/views/dependencyClassification.scala.html @@ -0,0 +1,40 @@ +@( + prefix: String, + dependencies: Seq[(Int, Library)], + allTags: Seq[(Int, LibraryTag)], + dependenciesTags: Map[Int, Set[LibraryTagAssignment]], + details: (Int, PlainLibraryIdentifier) => Html +) +
    +@for((libraryId, Library(lib, classified)) <- dependencies){ +
  • + @lib.libraryType: @lib.libraryIdentifier + + @if(lib.libraryType == LibraryType.Maven){ + mvnrepository.com» + libraries.io» + @defining(lib.libraryIdentifier.takeWhile(_ != ':').split('.')){ reverseDomain => + @if(! reverseDomain.startsWith(Seq("javax")) ){ + @for(i <- reverseDomain.length to 2 by -1; guessedDomain = reverseDomain.take(i).reverse.mkString("."); if !(TooGenericDomains contains guessedDomain)){ + @guessedDomain» + } + } + } + } + Google» + +
    + @details(libraryId, lib) + @defining(dependenciesTags.getOrElse(libraryId, Set.empty)) { libraryTags => + @for( + (tagId, tag) <- allTags.sortBy(_._2.name.toLowerCase); + exists = libraryTags.map(_.tagId) contains tagId + ){ + + } + } + +
    +
  • +} +
diff --git a/app/views/dependencyList.scala.html b/app/views/dependencyList.scala.html new file mode 100644 index 0000000..1d29e3d --- /dev/null +++ b/app/views/dependencyList.scala.html @@ -0,0 +1,77 @@ +@(idPrefix: String, list: Seq[GroupedDependency], selectorOption: Option[String], expandByDefault: Boolean = true, addButtons: Boolean = true) +@cpeHtmlId(cpe: String) = @{ + cpe.getBytes("utf-8").mkString("-") +} + +@for(dep <- list; depPrefix = s"$idPrefix-${dep.sha1}"){ +

+ @libraryIdentification(dep, Some(cpe => s"$idPrefix-${dep.sha1}-suppression-cpe-${cpeHtmlId(cpe)}"), addLink = false, addButtons = addButtons) + @for(s <- dep.maxCvssScore) { + + (@s + × @dep.projects.size + = @dep.ysdssScore) + (vulns: @dep.vulnerabilities.size) + + } + @dep.cpeIdentifiers.toSeq match { + case Seq() => {} + case cpeIds => { + + } + } +

+ @for(identifier <- dep.identifiers; cpe <- identifier.toCpeIdentifierOption ) { +
@SuppressionXml.forCpe(dep, cpe)
+ } +
+ @if(dep.descriptions.size > 1){ +
Multiple descriptions for this dependency!
+ } + @for(descriptionParagraphs <- dep.parsedDescriptions){ +
+ @for(descriptionParagraphLines <- descriptionParagraphs){ +

+ @for(line <- descriptionParagraphLines) { + @line
+ } +

+ } +
+ } +

Evidence

+ + + + + + + + + @for(ev <- dep.dependencies.keySet.map(_.evidenceCollected).flatten){ + + + } +
confidenceevidence typenamesourcevalue
@ev.confidence + @ev.evidenceType + @ev.name + @ev.source + @ev.value +
+

Affected projects (@dep.projects.size)

+
    @for(p <- dep.projects.toIndexedSeq.sorted){
  • @friendlyProjectName(p)
  • }
+

Vulnerabilities (@dep.vulnerabilities.size)

+
    + @for(vuln <- dep.vulnerabilities.toSeq.sortBy(_.cvssScore.map(-_)); vulnPrefix = s"$depPrefix-vulnerabilities-details-${vuln.name}"){ +
  • +
    @vuln.name
    +
    + @vulnerability("h6", s"$idPrefix-${dep.sha1}", vuln) +
    CVE suppression
    +
    @SuppressionXml.forVuln(dep, vuln)
    +
    +
  • + } +
+
+} diff --git a/app/views/filters/all.scala.html b/app/views/filters/all.scala.html new file mode 100644 index 0000000..7dae26d --- /dev/null +++ b/app/views/filters/all.scala.html @@ -0,0 +1 @@ +all \ No newline at end of file diff --git a/app/views/filters/project.scala.html b/app/views/filters/project.scala.html new file mode 100644 index 0000000..451c6b3 --- /dev/null +++ b/app/views/filters/project.scala.html @@ -0,0 +1,2 @@ +@(currentProject: ReportInfo) +Project: @friendlyProjectName(currentProject) \ No newline at end of file diff --git a/app/views/filters/team.scala.html b/app/views/filters/team.scala.html new file mode 100644 index 0000000..cfe1d70 --- /dev/null +++ b/app/views/filters/team.scala.html @@ -0,0 +1,2 @@ +@(teamId: String) +Team: @teamId \ No newline at end of file diff --git a/app/views/genericSection.scala.html b/app/views/genericSection.scala.html new file mode 100644 index 0000000..471266a --- /dev/null +++ b/app/views/genericSection.scala.html @@ -0,0 +1,3 @@ +@(idPrefix: String)(ht: String)(name: String, description: String)(content: Html) +<@ht id="@idPrefix-@name-header" data-toggle="collapse" data-target="#@idPrefix-@name-details">@description +
@content
diff --git a/app/views/groupedDependencyList.scala.html b/app/views/groupedDependencyList.scala.html new file mode 100644 index 0000000..646ad9e --- /dev/null +++ b/app/views/groupedDependencyList.scala.html @@ -0,0 +1,12 @@ +@(name: String, id: String, collapse: Boolean = false, allowSnoozes: Boolean = true, versions: Map[String, Int])(list: Seq[GroupedDependency])(implicit rh: DefaultRequest, snoozes: SnoozesInfo, messages: Messages) +@conditionalList(list, name, id, collapse = collapse, allowSnoozes = allowSnoozes, versions = versions){ + + @for(dep <- list){ + + + + + + } +
@identifiers(dep.mavenIdentifiers)@identifiers(dep.cpeIdentifiers)@dep.descriptions
+} \ No newline at end of file diff --git a/app/views/html/SuppressionXml.scala b/app/views/html/SuppressionXml.scala new file mode 100644 index 0000000..d5bd7d0 --- /dev/null +++ b/app/views/html/SuppressionXml.scala @@ -0,0 +1,10 @@ +package views.html + +import com.ysoft.odc.{GroupedDependency, Vulnerability} +object SuppressionXml { + + def forCpe(dep: GroupedDependency, cpe: String) = suppressionXmlPre(dep, {cpe}) + + def forVuln(dep: GroupedDependency, vuln: Vulnerability) = suppressionXmlPre(dep, {vuln.name}) + +} diff --git a/app/views/html/package.scala b/app/views/html/package.scala new file mode 100644 index 0000000..e1b7ef0 --- /dev/null +++ b/app/views/html/package.scala @@ -0,0 +1,9 @@ +package views + +import models.User + +package object html { + type SortedMap[A, B] = scala.collection.SortedMap[A, B] + type UserAwareRequest[T] = controllers.AuthenticatedController#UserAwareRequest[T] + type DefaultRequest = UserAwareRequest[_] +} diff --git a/app/views/identifier.scala.html b/app/views/identifier.scala.html new file mode 100644 index 0000000..d40ebca --- /dev/null +++ b/app/views/identifier.scala.html @@ -0,0 +1,3 @@ +@(identifier: Identifier, addLink: Boolean = true) +@identifier.confidence.toString.toLowerCase: +@secureLink(if(addLink) identifier.url else ""){@identifier.name} \ No newline at end of file diff --git a/app/views/identifiers.scala.html b/app/views/identifiers.scala.html new file mode 100644 index 0000000..30f06dc --- /dev/null +++ b/app/views/identifiers.scala.html @@ -0,0 +1,7 @@ +@(identifiers: Set[Identifier]) + +@for(i <- identifiers){ +
+ @identifier(i) +
+} \ No newline at end of file diff --git a/app/views/index.scala.html b/app/views/index.scala.html new file mode 100644 index 0000000..24b1306 --- /dev/null +++ b/app/views/index.scala.html @@ -0,0 +1,88 @@ +@( + vulnerableDependencies: Seq[GroupedDependency], + unclassifiedDependencies: Seq[(Int, Library)], + warnings: Seq[Warning], + groupedDependencies: Seq[GroupedDependency], + dependenciesForLibraries: Map[PlainLibraryIdentifier, Set[GroupedDependency]], + allTags: Seq[(Int, LibraryTag)], + relatedDependenciesTags: Map[Int, Set[LibraryTagAssignment]], + librariesForTagsWithWarning: SortedMap[(Int, LibraryTag), Seq[(Int, Library)]], + lastRefreshTime: DateTime, + versions: Map[String, Int] +)(implicit req: DefaultRequest, snoozes: SnoozesInfo , messages: Messages) +@import com.ysoft.odc.Confidence +@import helper._ + +@main("Y Soft Dependency status"){ + +@form(routes.Application.purgeCache(versions, "index")){ + @CSRF.formField + (last update form build server: @lastRefreshTime) +} + +@conditionalList(warnings, "Warnings", "warnings", allowSnoozes = false, versions = versions){ + @for(w <- warnings.sortBy(w => (-w.severity.id, w.id)); isLow <- Some(w.severity < WarningSeverity.Warning) ){ + @defining(snoozes(s"warning-${w.id}")){ si => +
"alert-warning" + case controllers.WarningSeverity.Info => "alert-info" + }) @if(si.isSnoozed){ text-muted}" id="warning-@w.id"> + + @snoozeButton(s"warning-${w.id}", si, collapseByDefault = false) + @if(w.allowSnoozes){ + @snoozeForm(s"warning-${w.id}", si, versions) + } +
+ @w.html + @if(w.allowSnoozes){ + @snoozesList(s"warning-${w.id}", si, versions) + } +
+
+
+ } + } +} + +@conditionalList(vulnerableDependencies, s"Vulnerable dependencies ${(Seq(None) ++ vulnerableDependencies.map(_.maxCvssScore)).max.map{maxScore => s"(max CVSS: $maxScore)"}.getOrElse("")}", "vulnerable", versions = versions) { + @dependencyList("vulnerable", vulnerableDependencies.sortBy(d => (d.maxCvssScore.map(-_), d.cpeIdentifiers.map(_.toCpeIdentifierOption.get).toSeq.sorted.mkString(" "))), None) +} +@* groupedDependencyList("Unclassified dependencies", "unclassified")(unclassifiedDependencies) *@ + +@conditionalList(unclassifiedDependencies, "Unclassified dependencies", "unclassified", versions = versions){ + @dependencyClassification( + prefix = "unclassified-dependency", + dependencies = unclassifiedDependencies, + allTags = allTags, + dependenciesTags = relatedDependenciesTags, + details = (libraryId: Int, lib: PlainLibraryIdentifier) => { + dependenciesForLibraries.get(lib).fold{Html("

No details

")} { deps => + dependencyList(s"unclassified-library-$libraryId-details", deps.toSeq /*TODO: sort */, None) + } + } + ) +} + +@groupedDependencyList("Dependencies with low confidence of GAV", "gav-low-confidence", versions = versions)(groupedDependencies.filter(_.mavenIdentifiers.exists(_.confidence < Confidence.High))) + +@for(((tagId, tag), libraries) <- librariesForTagsWithWarning){ + @conditionalList(libraries, s"${tag.name}", s"tag-warning-$tagId", versions = versions){ + @for(note <- tag.note){ +

@note

+ } + @dependencyClassification( + prefix = s"tag-warning-$tagId-list", + dependencies = libraries, + allTags = allTags, + dependenciesTags = relatedDependenciesTags, + details = (_, _) => Html("") + ) + } +} + +@* @groupedDependencyList("All dependencies", "all", collapse = true)(groupedDependencies) *@ + +} diff --git a/app/views/libraryIdentification.scala.html b/app/views/libraryIdentification.scala.html new file mode 100644 index 0000000..8eeac78 --- /dev/null +++ b/app/views/libraryIdentification.scala.html @@ -0,0 +1,16 @@ +@(dep: GroupedDependency, suppressionXmlIdOption: Option[String => String] = None, addLink: Boolean = true, addButtons: Boolean = true) +@import com.ysoft.odc.Confidence +@import scala.math.Ordered.orderingToOrdered + +@if(!dep.identifiers.exists(_.confidence >= Confidence.High)){ + file: @dep.fileNames.toSeq.sorted.mkString(", ")@if(addButtons){ } +} +@for(id <- dep.identifiers.toSeq.sortBy(i => (i.confidence, i.identifierType, i.name, i.url)).reverse){ + + @identifier(id, addLink) + @for(cpe <- id.toCpeIdentifierOption; suppressionXmlId <- suppressionXmlIdOption; if addButtons){ + + } + @if(addButtons && suppressionXmlIdOption.isDefined){ } + +} diff --git a/app/views/main.scala.html b/app/views/main.scala.html new file mode 100644 index 0000000..fea26ce --- /dev/null +++ b/app/views/main.scala.html @@ -0,0 +1,96 @@ +@import helper._ +@(title: String, headExtension: Html = Html(""), projectsOption: Option[(ProjectsWithSelection, Option[String] => Call)] = None)(content: Html)(implicit header: DefaultRequest) + + + + @title + + + + + + + + + + + + @if(!header.secure){ + + } + @headExtension + + + + + +
+

@title

+ @content + +
+ That's all +
+ + + \ No newline at end of file diff --git a/app/views/secureLink.scala.html b/app/views/secureLink.scala.html new file mode 100644 index 0000000..dcc71ac --- /dev/null +++ b/app/views/secureLink.scala.html @@ -0,0 +1,5 @@ +@(url: String)(content: Html) +@url match{ + case NormalUrlPattern(_ @ _*) => {@content} + case "" => {@content} +} diff --git a/app/views/snoozeButton.scala.html b/app/views/snoozeButton.scala.html new file mode 100644 index 0000000..e8f859a --- /dev/null +++ b/app/views/snoozeButton.scala.html @@ -0,0 +1,4 @@ +@(id: String, si: SnoozeInfo, collapseByDefault: Boolean) + \ No newline at end of file diff --git a/app/views/snoozeForm.scala.html b/app/views/snoozeForm.scala.html new file mode 100644 index 0000000..688df4a --- /dev/null +++ b/app/views/snoozeForm.scala.html @@ -0,0 +1,9 @@ +@(id: String, si: SnoozeInfo, versions: Map[String, Int])(implicit rh: RequestHeader, snoozes: SnoozesInfo, messages: Messages) +@import helper._ + +@form((routes.Application.snooze(id, versions): Call).withFragment(id), 'id -> s"$id-snoozing", 'class -> s"snoozing collapse${if(si.shouldExpandForm) "in" else ""}") { + @CSRF.formField + @inputText(si.form("until"), '_label -> "Snooze until", Symbol("data-provide") -> "datepicker", Symbol("data-date-format") -> "dd-mm-yyyy") + @inputText(si.form("reason"), '_label -> "Reason") + +} diff --git a/app/views/snoozesList.scala.html b/app/views/snoozesList.scala.html new file mode 100644 index 0000000..1ba3743 --- /dev/null +++ b/app/views/snoozesList.scala.html @@ -0,0 +1,16 @@ +@(unused_id: String, si: SnoozeInfo, versions: Map[String, Int])(implicit requestHeader: RequestHeader) +@import helper._ +@if(si.isSnoozed){ +

Snooze details

+
    + @for((snoozeId, snooze) <- si.snoozes){ +
  • + @form(routes.Application.unsnooze(snoozeId, versions)) { + @snooze.reason – until @snooze.until + @CSRF.formField + + } +
  • + } +
+} \ No newline at end of file diff --git a/app/views/statistics/allLibraries.scala.html b/app/views/statistics/allLibraries.scala.html new file mode 100644 index 0000000..a9e89ce --- /dev/null +++ b/app/views/statistics/allLibraries.scala.html @@ -0,0 +1,18 @@ +@( + projectsWithSelection: ProjectsWithSelection, + allDependencies: Seq[GroupedDependency] +)(implicit header: DefaultRequest) + +@main( + title = s"All libraries for ${projectsWithSelection.projectNameText}", + projectsOption = Some((projectsWithSelection, routes.Statistics.allLibraries(_))) +){ + @dependencyList( + "all", + allDependencies.sortBy(_.identifiers.toIndexedSeq.sortBy(i => (i.confidence.id, i.identifierType, i.name)).mkString(", ")), + selectorOption = projectsWithSelection.selectorString, + expandByDefault = false, + addButtons = false + ) + +} \ No newline at end of file diff --git a/app/views/statistics/basic.scala.html b/app/views/statistics/basic.scala.html new file mode 100644 index 0000000..96242dc --- /dev/null +++ b/app/views/statistics/basic.scala.html @@ -0,0 +1,187 @@ +@( + projectsWithSelection: ProjectsWithSelection, + tagStatistics: Seq[Statistics.TagStatistics], + parsedReports: DependencyCheckReportsParser.Result +)(implicit messagesApi: MessagesApi, requestHeader: DefaultRequest) +@import com.ysoft.odc.CWE +@import controllers.Statistics.TagStatistics +@import play.api.libs.json.{JsNull, JsString} +@import scala.language.implicitConversions +@implicitTagStatistics(ts: TagStatistics) = @{ts.stats} +@he = { + + + + + + + + + +} +@plotData(frequency: Map[Option[CWE], Int]) = @{ + import play.api.libs.json.Json._ + val (ticks, details, values) = frequency.toSeq.sortBy{case (cweOption, _) => cweOption.map(c => c.numberOption -> c.name)}.map{ + case (Some(cwe), freq) => (toJson(cwe.brief), toJson(cwe.name), freq) + case (None, freq) => (JsString("(none)"), JsNull, freq) + }.unzip3 + toJson(Map( + "ticks" -> toJson(ticks), + "details" -> toJson(details), + "values" -> toJson(values) + )) +} +@main( + title = s"statistics for ${projectsWithSelection.projectNameText}", + headExtension = he, + projectsOption = Some((projectsWithSelection, routes.Statistics.basic(_))) +){ + + All dependencies: @parsedReports.groupedDependencies.size
+ Vulnerable dependencies: @parsedReports.vulnerableDependencies.size
+ Vulnerabilities: @parsedReports.vulnerableDependencies.flatMap(_.vulnerabilities.map(_.name)).toSet.size
+ Unique CPEs of vulnerable dependencies: @parsedReports.vulnerableDependencies.flatMap(_.cpeIdentifiers.map(_.toCpeIdentifierOption.get)).toSet.size
+ Unique CPEs of all dependencies: @parsedReports.groupedDependencies.flatMap(_.cpeIdentifiers.map(_.toCpeIdentifierOption.get)).toSet.size
+ @if(!projectsWithSelection.isProjectSpecified){ + Multi-project dependencies: @parsedReports.groupedDependencies.filter(_.projects.size > 1).toSet.size
+ } + + +
+ + + + + + + + + + + + + + + + @for(s <- tagStatistics){ + + + + + + + + + + } + +
tag name# of vulnsvulnerableallvulnerable/allCPE %vulnerable/CPE
+ @s.tag.name + @*
@s.tag.note
*@ +
+ @s.vulnerabilities.size + + @s.vulnerableDependencies.size@s.dependencies.size@(f"${s.vulnerableRatio*100}%2.2f") %@(f"${s.cpeRatio*100}%2.2f") %@(f"${s.vulnerableDependencies.size.toDouble*100.0/s.dependenciesWithCpe.size.toDouble}%2.2f") %
+} \ No newline at end of file diff --git a/app/views/statistics/vulnerabilities.scala.html b/app/views/statistics/vulnerabilities.scala.html new file mode 100644 index 0000000..578bd50 --- /dev/null +++ b/app/views/statistics/vulnerabilities.scala.html @@ -0,0 +1,40 @@ +@( + projectsWithSelection: ProjectsWithSelection, + tagOption: Option[(Int, LibraryTag)], + statistics: Statistics.LibDepStatistics +)(implicit messagesApi: MessagesApi, requestHeader: DefaultRequest) + +@main( + title = s"details for ${projectsWithSelection.projectNameText}${tagOption.map(_._2.name).fold("")(" and tag "+_)}", + projectsOption = Some((projectsWithSelection, x => routes.Statistics.vulnerabilities(x, tagOption.map(_._1)))) +){ + We have @statistics.vulnerabilitiesToDependencies.size vulnerabilities + of @statistics.vulnerabilitiesToDependencies.flatMap(_._2).toSet.size dependencies (@statistics.vulnerabilitiesToDependencies.flatMap(_._2.flatMap(_.plainLibraryIdentifiers)).toSet.size libaries). + @if(!projectsWithSelection.isProjectSpecified){ + They are affecting @statistics.vulnerabilitiesToDependencies.flatMap(_._2.flatMap(_.projects)).toSet.size projects. + }else{ + Just one project is selected. +
When a project is selected, YSVSS might differ, as it is computed over a subset of subprojects. As a result, order of vulnerabilities might differ from their order at all-projects view.
+ } +
+ Vulnerabilities are sorted by number of affected projects multiplied by their severity. If the score is the same, then they are sorted by severity. If even this matches, they are sorted by name (which is related to vulnerability age). +
+ @for((vulnerability, dependencies) <- statistics.vulnerabilitiesToDependencies.toSeq.sortBy{case (vuln, deps) => + ( + vuln.ysvssScore(deps).map(-_), // total score + vuln.cvssScore.map(-_), // CVSS score + vuln.name // make it deterministic + ) + }){ +

@vulnerability.name + + (@(vulnerability.cvss.score.getOrElse{"?"}) × + @dependencies.flatMap(_.projects).toSet.size = + @(vulnerability.ysvssScore(dependencies).fold{"?"}{d => f"$d%2.2f"}) + ) +

+

@vulnerability.description

+ @*

@dependencies.map(_.identifiers)

*@ + @*

@dependencies.flatMap(_.projects).toSet

*@ + } +} \ No newline at end of file diff --git a/app/views/statistics/vulnerabilitiesForLibrary.scala.html b/app/views/statistics/vulnerabilitiesForLibrary.scala.html new file mode 100644 index 0000000..dc24168 --- /dev/null +++ b/app/views/statistics/vulnerabilitiesForLibrary.scala.html @@ -0,0 +1,78 @@ +@( + vulnsAndVersionOption: Option[(Traversable[Vulnerability], String)], + cpes: Seq[String], + isDbOld: Boolean +)(implicit header: DefaultRequest) +@import helper._ +@main( + title = "Vulnerabilities for a libary" +){ + + @form(routes.Statistics.searchVulnerableSoftware(Seq(), None), 'onsubmit-> + """ + |return (function(f){ + | var selectedCpes = $(f.elements.versionlessCpes).filter(function(i, x){return x.checked;}).map(function(i, x){return x.value;}).toArray() + | if(selectedCpes.length == 0){ + | alert("Choose at least one CPE, please!"); + | return false; + | } + |})(this); + |""".stripMargin + ){ + +
    + @for(cpe <- cpes){ +
  • + } +
+ + } + @if(isDbOld){ +
The vulnerability database seems to be outdated. Result might be thus inaccurate. Contact the administrator, please.
+ } + @vulnsAndVersionOption.fold{ + Select desired version, please + }{ case (vulns, version) => + @if(vulns.isEmpty){ +
No known vulnerabilities for version @version.
+ }else{ +
There @if(vulns.size == 1){is one known vulnerability}else{are some known vulnerabilities} for version @version. Consider @if(vulns.size==1){its}else{their} impact before using the library, please.
+ @for(vuln <- vulns.toIndexedSeq.sortBy(v => (v.cvssScore.map(-_), v.name))){ +

@vuln.name

+ @vulnerability("h3", s"vulnerability-${vuln.name}-details", vuln) + } + } + } + + @*if(vulnsAndVersionOption.isEmpty){ *@ + + @* } *@ + +} \ No newline at end of file diff --git a/app/views/statistics/vulnerability.scala.html b/app/views/statistics/vulnerability.scala.html new file mode 100644 index 0000000..e05d11b --- /dev/null +++ b/app/views/statistics/vulnerability.scala.html @@ -0,0 +1,43 @@ +@( + projectsWithSelection: ProjectsWithSelection, + vulnerability: Vulnerability, + affectedProjects: Map[ReportInfo, Set[GroupedDependency]], + vulnerableDependencies: Set[GroupedDependency], + affectedLibraries: Set[PlainLibraryIdentifier] +)(implicit header: DefaultRequest) +@section = @{views.html.genericSection("vuln")("h2") _} +@main( + title = s"vulnerability ${vulnerability.name} for ${projectsWithSelection.projectNameText}", + projectsOption = Some((projectsWithSelection, p => routes.Statistics.vulnerability(vulnerability.name, p))) +) { + @if(projectsWithSelection.isProjectSpecified){ +
The vulnerability details are limited to some subset of projects.
Show it for all projects!
+ } + @section("details", "Vulnerability details") { + @views.html.vulnerability("h2", "vuln-details", vulnerability) + } + @section("affected-libs", s"Unique affected libraries – without version number (${affectedLibraries.size})"){ +
    + @for(lib <- affectedLibraries){ +
  • @lib
  • + } +
+ } + @section("affected-deps", s"Unique affected dependencies (${vulnerableDependencies.size})"){ +
    + @for(dep <- vulnerableDependencies){ +
  • @libraryIdentification(dep)
  • + } +
+ } + @section("affected-projects", s"Affected projects (${affectedProjects.size} projects with ${affectedProjects.flatMap(_._2).size} occurrences)"){ + @for((project, dependencies) <- affectedProjects.toSeq.sortBy(_._1)){ +

@friendlyProjectName(project) (@dependencies.size)

+
    + @for(dep <- dependencies.toSeq){ +
  • @libraryIdentification(dep)
  • + } +
+ } + } +} diff --git a/app/views/statistics/vulnerabilityNotFound.scala.html b/app/views/statistics/vulnerabilityNotFound.scala.html new file mode 100644 index 0000000..616d45c --- /dev/null +++ b/app/views/statistics/vulnerabilityNotFound.scala.html @@ -0,0 +1,24 @@ +@( + projectsWithSelection: ProjectsWithSelection, + name: String +)(implicit header: DefaultRequest) + +@main( + title = s"Unknown vulnerability $name for ${projectsWithSelection.projectNameText}", + projectsOption = Some((projectsWithSelection, p => routes.Statistics.vulnerability(name, p))) +){ +
Vulnerability @name is not found@if(projectsWithSelection.isProjectSpecified){ for selected project(s)}.
+

Possible solutions

+
    + @if(projectsWithSelection.isProjectSpecified){ +
  • + Maybe the vulnerability does not affect this project, but it might affect other projects.
    + Look at all the projects! +
  • + } +
  • + Maybe the vulnerability does not affect any of the projects.
    + Look at NVD +
  • +
+} \ No newline at end of file diff --git a/app/views/statistics/vulnerableLibraries.scala.html b/app/views/statistics/vulnerableLibraries.scala.html new file mode 100644 index 0000000..1565796 --- /dev/null +++ b/app/views/statistics/vulnerableLibraries.scala.html @@ -0,0 +1,64 @@ +@( + projectsWithSelection: ProjectsWithSelection, + vulnerableDependencies: Seq[GroupedDependency], + allDependenciesCount: Int +)(implicit header: DefaultRequest) + +@main( + title = s"Vulnerable libraries for ${projectsWithSelection.projectNameText} (${vulnerableDependencies.size} deps, ${vulnerableDependencies.flatMap(_.cpeIdentifiers.map(_.toCpeIdentifierOption.get)).toSet.size} CPEs)", + projectsOption = Some((projectsWithSelection, routes.Statistics.vulnerableLibraries(_))) +){ + + +

Plot

+
+ +

List

+
+

Libraries are sorted:

+
    +
  1. by total score (max vulnerability score × number of affected dependencies) if vulnerability score is defined for at least one vulnerability
  2. +
  3. by affected dependency count if the score above is not defined
  4. +
  5. by number of vulnerabilities
  6. +
  7. by affected project count
  8. +
+

Note that the number of affected projects is calculated from the current view, not from all projects (unless all projects are selected).

+
+ @dependencyList( + "vulnerable", + vulnerableDependencies.sortBy(d => ( + d.ysdssScore.map(-_), // total score is the king + if(d.ysdssScore.isEmpty) Some(-d.dependencies.size) else None, // more affected dependencies if no vulnerability has defined severity + -d.vulnerabilities.size, // more vulnerabilities + -d.projects.size, // more affected projects + d.cpeIdentifiers.map(_.toCpeIdentifierOption.get).toSeq.sorted.mkString(" ")) // at least make the order deterministic + ), + selectorOption = projectsWithSelection.selectorString, + expandByDefault = false, + addButtons = false + ) +} \ No newline at end of file diff --git a/app/views/suppressionXmlPre.scala.html b/app/views/suppressionXmlPre.scala.html new file mode 100644 index 0000000..6a29a73 --- /dev/null +++ b/app/views/suppressionXmlPre.scala.html @@ -0,0 +1,12 @@ +@(dep: GroupedDependency, details: scala.xml.Node) +
<?xml version="1.0" encoding="UTF-8"?>
+@((
+
+    
+        file name: {dep.fileNames.mkString(" OR ")}
+        {dep.sha1}
+        {details}
+    
+
+).toString)
+
\ No newline at end of file diff --git a/app/views/tagsImport.scala.html b/app/views/tagsImport.scala.html new file mode 100644 index 0000000..0390815 --- /dev/null +++ b/app/views/tagsImport.scala.html @@ -0,0 +1,9 @@ +@(f: Form[String])(implicit requestHeader: DefaultRequest, messages: Messages) +@import helper._ +@main("Data import"){ + @form(action = controllers.routes.Application.tagsImportAction()){ + @CSRF.formField + @textarea(f("data")) + + } +} \ No newline at end of file diff --git a/app/views/vulnerability.scala.html b/app/views/vulnerability.scala.html new file mode 100644 index 0000000..9ecb385 --- /dev/null +++ b/app/views/vulnerability.scala.html @@ -0,0 +1,35 @@ +@(ht: String, idPrefix: String, vuln: Vulnerability) +@row[T](name: String, render: T => String = {(_:T).toString})(valueOption: Option[T]) = { + @for(value <- valueOption){ + + @name + @render(value) + + } +} +@section = @{views.html.genericSection(idPrefix)(ht) _} + + @row("CWE")(vuln.cweOption) + @row("CVSS: score")(vuln.cvss.score) + @row("CVSS: authenticationr")(vuln.cvss.authenticationr) + @row("CVSS: availability impact")(vuln.cvss.availabilityImpact) + @row("CVSS: access vector")(vuln.cvss.accessVector) + @row("CVSS: integrity impact")(vuln.cvss.integrityImpact) + @row("CVSS: access complexity")(vuln.cvss.accessComplexity) + @row("CVSS: confidential impact")(vuln.cvss.confidentialImpact) +
+@vuln.description +@section("vuln-sw", "Vulnerable software"){ +
    + @for(sw <- vuln.vulnerableSoftware){ +
  • @sw.name@if(sw.allPreviousVersion){ and all previous versions}
  • + } +
+} +@section("references", "References"){ +
    + @for(reference <- vuln.references){ +
  • @secureLink(reference.url){@reference.source: @reference.name}
  • + } +
+} diff --git a/app/views/warnings/badGroupedDependencies.scala.html b/app/views/warnings/badGroupedDependencies.scala.html new file mode 100644 index 0000000..cdb7b42 --- /dev/null +++ b/app/views/warnings/badGroupedDependencies.scala.html @@ -0,0 +1,7 @@ +@(name: String, count: Int, items: Traversable[_]) +@name (@count)
+
    + @for(i <- items){ +
  • @i
  • + } +
diff --git a/app/views/warnings/badValues.scala.html b/app/views/warnings/badValues.scala.html new file mode 100644 index 0000000..b44e461 --- /dev/null +++ b/app/views/warnings/badValues.scala.html @@ -0,0 +1,7 @@ +@(name: String, items: Seq[(ReportInfo, Html)]) +@name: +
    + @for((name, i) <- items){ +
  • @friendlyProjectName(name): @i
  • + } +
\ No newline at end of file diff --git a/app/views/warnings/emptyResults.scala.html b/app/views/warnings/emptyResults.scala.html new file mode 100644 index 0000000..6eafc9b --- /dev/null +++ b/app/views/warnings/emptyResults.scala.html @@ -0,0 +1,5 @@ +@(emptyReports: Seq[Build], urlBase: String) +Following projects have produced no results: +@for(build <- emptyReports.toSeq.sortBy(_.projectName)){ +
  • @secureLink(build.resultLink(urlBase)){@build.projectName}
  • +} diff --git a/app/views/warnings/failedReports.scala.html b/app/views/warnings/failedReports.scala.html new file mode 100644 index 0000000..b839b2b --- /dev/null +++ b/app/views/warnings/failedReports.scala.html @@ -0,0 +1,7 @@ +@(failedReports: Set[Build], urlBase: String) +There are some reports that failed to build: +
      + @for(build <- failedReports.toSeq.sortBy(_.projectName)){ +
    • @secureLink(build.resultLink(urlBase)){@build.projectName}
    • + } +
    \ No newline at end of file diff --git a/app/views/warnings/failedResults.scala.html b/app/views/warnings/failedResults.scala.html new file mode 100644 index 0000000..297925a --- /dev/null +++ b/app/views/warnings/failedResults.scala.html @@ -0,0 +1,19 @@ +@(errors: Map[String, Throwable]) +
      + Some reports failed to be downloaded: + @for((project, e) <- errors){ +
    • + @project: @e + @(e match { + case upickle.Invalid.Data(data, msg) => s"$msg (data: $data)" + case upickle.Invalid.Json(msg, input) => s"$msg (input: $input)" + case _ => "" + }) + +
    • + @{ + play.api.Logger.logger.error(s"Project results of $project failed to parse.", e) + () + } + } +
    \ No newline at end of file diff --git a/app/views/warnings/groupedDependencies.scala.html b/app/views/warnings/groupedDependencies.scala.html new file mode 100644 index 0000000..3d92709 --- /dev/null +++ b/app/views/warnings/groupedDependencies.scala.html @@ -0,0 +1,10 @@ +@(items: IndexedSeq[GroupedDependency])(implicit rh: DefaultRequest, snoozes: SnoozesInfo, messages: Messages) +(ignore this item) +@groupedDependencyList(name = "", id = s"grouped-dependencies-warning-${java.util.UUID.randomUUID.toString}", collapse = false, allowSnoozes = false, versions = Map())(list = items) +@for(groupedDependency <- items){ +
  • + @groupedDependency.fileNames.toSeq.sorted + @identifiers(groupedDependency.identifiers) + @groupedDependency.dependencies.keySet.groupBy(_.evidenceCollected) +
  • +} \ No newline at end of file diff --git a/app/views/warnings/resultsWithErrorMessages.scala.html b/app/views/warnings/resultsWithErrorMessages.scala.html new file mode 100644 index 0000000..efb93b0 --- /dev/null +++ b/app/views/warnings/resultsWithErrorMessages.scala.html @@ -0,0 +1,5 @@ +@(reportsWithErrorMessages: Seq[Build], urlBase: String) +Following projects seem to contain some error messages in their logs: +@for(build <- reportsWithErrorMessages.toSeq.sortBy(_.projectName)){ +
  • @secureLink(build.resultLink(urlBase)){@build.projectName}
  • +} diff --git a/app/views/warnings/textWarning.scala.html b/app/views/warnings/textWarning.scala.html new file mode 100644 index 0000000..dc0111b --- /dev/null +++ b/app/views/warnings/textWarning.scala.html @@ -0,0 +1,2 @@ +@(text: String) +@text \ No newline at end of file diff --git a/app/views/warnings/unknownIdentifierType.scala.html b/app/views/warnings/unknownIdentifierType.scala.html new file mode 100644 index 0000000..f067544 --- /dev/null +++ b/app/views/warnings/unknownIdentifierType.scala.html @@ -0,0 +1,7 @@ +@(unknownIdentifierTypes: Set[String]) +There are some unknown identifier types. These will not be handled by the application: +
      + @for(identifierType <- unknownIdentifierTypes.toSeq.sorted){ +
    • @identifierType
    • + } +
    \ No newline at end of file diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..62dea65 --- /dev/null +++ b/build.gradle @@ -0,0 +1,9 @@ +task buildPlayApp(type:Exec){ + //environment("JAVA_HOME", System.getProperty("java.home")) + if(System.getProperty("os.name").toLowerCase().startsWith("win")){ + // unsupported OS because of silent fails; commandLine 'cmd', '/c', 'activator', 'stage' + commandLine 'report_error' + }else{ + commandLine './activator', 'clean', 'dist' + } +} diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..accf104 --- /dev/null +++ b/build.sbt @@ -0,0 +1,97 @@ +name := """odc-analyzer""" + +version := "1.0-SNAPSHOT" + +lazy val root = (project in file(".")).enablePlugins(PlayScala) + +scalaVersion := "2.11.7" + +resolvers += "Atlassian Releases" at "https://maven.atlassian.com/public/" + +libraryDependencies ++= Seq( + //jdbc, + cache, + ws, + filters + //specs2 % Test +) + +//resolvers += "scalaz-bintray" at https?"http://dl.bintray.com/scalaz/releases" + +libraryDependencies += "com.lihaoyi" %% "upickle" % "0.3.4" + +//libraryDependencies += "com.typesafe.play" %% "play-ws" % "2.4.2" + +libraryDependencies += "com.jsuereth" %% "scala-arm" % "1.4" + +libraryDependencies += "org.ccil.cowan.tagsoup" % "tagsoup" % "1.2.1" + +libraryDependencies += "com.typesafe.play" %% "play-slick" % "1.1.1" + +libraryDependencies += "com.typesafe.play" %% "play-slick-evolutions" % "1.1.1" + +libraryDependencies += "com.github.tototoshi" %% "slick-joda-mapper" % "2.0.0" + +//libraryDependencies += "nu.validator.htmlparser" % "htmlparser" % "1.2.1" + +//libraryDependencies += "com.lihaoyi" %% "pprint" % "0.3.4" + +libraryDependencies += "com.github.nscala-time" %% "nscala-time" % "2.0.0" + +// libraryDependencies += "org.mariadb.jdbc" % "mariadb-java-client" % "1.1.9" + +libraryDependencies += "org.postgresql" % "postgresql" % "9.4-1201-jdbc41" + +libraryDependencies += "org.mariadb.jdbc" % "mariadb-java-client" % "1.3.3" + +libraryDependencies += "org.webjars" % "bootstrap" % "3.3.5" + +libraryDependencies += "org.webjars" % "jquery" % "2.1.4" + +libraryDependencies += "org.webjars" % "bootstrap-datepicker" % "1.4.0" + +libraryDependencies += "org.webjars" % "tablesorter" % "2.17.8" + +libraryDependencies += "org.webjars.bower" % "StickyTableHeaders" % "0.1.17" + +//libraryDependencies += "org.webjars.bower" % "plottable" % "1.5.0" + +//libraryDependencies += "org.webjars" % "d3js" % "3.5.6" + +libraryDependencies += "org.webjars" % "jqplot" % "1.0.8r1250" + +//libraryDependencies += "com.github.mumoshu" %% "play2-memcached-play24" % "0.7.0" + +libraryDependencies ++= Seq( + "com.mohiva" %% "play-silhouette" % "3.0.4", + "com.mohiva" %% "play-silhouette-testkit" % "3.0.4" % "test" +) + +libraryDependencies += "net.codingwell" %% "scala-guice" % "4.0.0" + +libraryDependencies += "net.ceedubs" %% "ficus" % "1.1.2" + +libraryDependencies += "org.owasp" % "dependency-check-core" % "1.3.0" + +routesImport += "binders.QueryBinders._" + +// Uncomment to use Akka +//libraryDependencies += "com.typesafe.akka" %% "akka-actor" % "2.3.11" + +// Play provides two styles of routers, one expects its actions to be injected, the +// other, legacy style, accesses its actions statically. +routesGenerator := InjectedRoutesGenerator + + +scalacOptions ++= Seq( + "-deprecation", // Emit warning and location for usages of deprecated APIs. + "-feature", // Emit warning and location for usages of features that should be imported explicitly. + "-unchecked", // Enable additional warnings where generated code depends on assumptions. + //"-Xfatal-warnings", // Fail the compilation if there are any warnings. + "-Xlint", // Enable recommended additional warnings. + "-Ywarn-adapted-args", // Warn if an argument list is modified to match the receiver. + "-Ywarn-dead-code", // Warn when dead code is identified. + "-Ywarn-inaccessible", // Warn about inaccessible types in method signatures. + "-Ywarn-nullary-override", // Warn when non-nullary overrides nullary, e.g. def foo() over def foo. + "-Ywarn-numeric-widen" // Warn when numerics are widened. +) \ No newline at end of file diff --git a/conf/application.conf.-example b/conf/application.conf.-example new file mode 100644 index 0000000..63495f6 --- /dev/null +++ b/conf/application.conf.-example @@ -0,0 +1,117 @@ +# This is the main configuration file for the application. +# ~~~~~ + +# Secret key +# ~~~~~ +# The secret key is used to secure cryptographics functions. +# +# This must be changed for production, but we recommend not changing it in this file. +# +# See https://www.playframework.com/documentation/latest/ApplicationSecret for more details. +play.crypto.secret = "changeme" + +# The application languages +# ~~~~~ +play.i18n.langs = [ "en" ] + +play.modules.enabled += "modules.ConfigModule" +play.modules.enabled += "modules.SilhouetteModule" + +yssdc{ + bamboo{ + url = … + } + reports { + provider = "bamboo" + bamboo{ + user = … + password = … + } + } + projects = {jobId:humanReadableName, …} + teams = […] + exclusions{ + missingGAV{ + bySha1 = [] + } + } + projectsToTeams = { + … + } + teamLeaders = { # all teams used here must be listed above + team: leader, + … + } +} + +# Router +# ~~~~~ +# Define the Router object to use for this application. +# This router will be looked up first when the application is starting up, +# so make sure this is the entry point. +# Furthermore, it's assumed your route file is named properly. +# So for an application router like `my.application.Router`, +# you may need to define a router file `conf/my.application.routes`. +# Default to Routes in the root package (and conf/routes) +# play.http.router = my.application.Routes + +# Database configuration +# ~~~~~ +# You can declare as many datasources as you want. +# By convention, the default datasource is named `default` +# + +slick.dbs.default { + # Connection to internal database. It must be PostgreSQL. + driver = "slick.driver.PostgresDriver$" + db{ + url = "jdbc:postgresql://localhost/odca" + user = … + password = … + } +} +slick.dbs.odc { + # Connection to ODC database. It should be MySQL/MariaDB. H2 DB is not supported. PostgreSQL might work if you get ODC working with it, Other databases might be supported in future. + driver = "slick.driver.MySQLDriver$" + db { + url = "jdbc:mysql://127.0.0.1/dependencycheck" + # Those credentials are default in ODC (but you might have changed them): + user = "dcuser" + password = "DC-Pass1337!" + } +} + +# Evolutions +# ~~~~~ +# You can disable evolutions if needed +# play.evolutions.enabled=false + +# You can disable evolutions for a specific datasource if necessary +# play.evolutions.db.default.enabled=false + +# If you want a persistent cache for development (it should speed up reload cycles), you might want to uncomment and adjust the following lines: +#play.modules.disabled+="play.api.cache.EhCacheModule" +#play.cache.path = "/home/user/.cache/odc-analysis" + + +silhouette { + # Authenticator settings + authenticator.cookieName = "authenticator" + authenticator.cookiePath = "/" + authenticator.secureCookie=false # is ignored; overriden in app/controllers/AuthController.scala; But it must be present! + authenticator.httpOnlyCookie = true + authenticator.useFingerprinting = true + authenticator.authenticatorIdleTimeout = 12 hours + authenticator.authenticatorExpiry = 12 hours + + authenticator.rememberMe.cookieMaxAge = 30 days + authenticator.rememberMe.authenticatorIdleTimeout = 5 days + authenticator.rememberMe.authenticatorExpiry = 30 days + + credentialsVerificationService{ + type="allow-all" # accepts any credentials; allowed in dev mode only + #type="external" # verifies credentials at the URL specified below + #url="http://localhost:9050/" + } +} + diff --git a/conf/evolutions/default/1.sql b/conf/evolutions/default/1.sql new file mode 100644 index 0000000..4ea067f --- /dev/null +++ b/conf/evolutions/default/1.sql @@ -0,0 +1,34 @@ +# --- !Ups + + +CREATE TABLE library ( + id SERIAL, + library_type VARCHAR(255) NOT NULL, -- We could use enums, but it is too much bothering in PostgreSQL. We'll enforce those constrainst on application level :) + identifier VARCHAR(255) NOT NULL, + classified BOOLEAN, + PRIMARY KEY (id) +); + +CREATE UNIQUE INDEX library_unique ON library (library_type, identifier); + +CREATE TABLE library_tag ( + id SERIAL, + name varchar(255) NOT NULL, + PRIMARY KEY (id) +); + +CREATE UNIQUE INDEX library_tag_unique ON library_tag (name); + +CREATE TABLE library_to_library_tag ( + library_id INTEGER NOT NULL REFERENCES library, + library_tag_id INTEGER NOT NULL REFERENCES library_tag, + context_dependent BOOLEAN +); + +CREATE UNIQUE INDEX library_to_library_tag_unique ON library_to_library_tag (library_id, library_tag_id); + +# --- !Downs + +DROP TABLE library; +DROP TABLE library_to_library_tag; +DROP TABLE library_tag; diff --git a/conf/evolutions/default/2.sql b/conf/evolutions/default/2.sql new file mode 100644 index 0000000..890706b --- /dev/null +++ b/conf/evolutions/default/2.sql @@ -0,0 +1,7 @@ +# --- !Ups + +ALTER TABLE library_tag ADD COLUMN note VARCHAR(1024) NULL DEFAULT NULL; + +# --- !Downs + +ALTER TABLE library_tag DROP COLUMN note; diff --git a/conf/evolutions/default/3.sql b/conf/evolutions/default/3.sql new file mode 100644 index 0000000..cc36555 --- /dev/null +++ b/conf/evolutions/default/3.sql @@ -0,0 +1,7 @@ +# --- !Ups + +ALTER TABLE library_tag ADD COLUMN warning_order INT NULL DEFAULT NULL; + +# --- !Downs + +ALTER TABLE library_tag DROP COLUMN warning_order; diff --git a/conf/evolutions/default/4.sql b/conf/evolutions/default/4.sql new file mode 100644 index 0000000..bc1134b --- /dev/null +++ b/conf/evolutions/default/4.sql @@ -0,0 +1,13 @@ +# --- !Ups + +CREATE TABLE snooze( + "id" SERIAL NOT NULL, + "until" DATE NOT NULL, + "snoozed_object_identifier" VARCHAR(512) NOT NULL, + "reason" VARCHAR(1024) NOT NULL +); +CREATE INDEX snooze_until ON snooze (until); + +# --- !Downs + +DROP TABLE snooze; \ No newline at end of file diff --git a/conf/evolutions/default/5.sql b/conf/evolutions/default/5.sql new file mode 100644 index 0000000..2016128 --- /dev/null +++ b/conf/evolutions/default/5.sql @@ -0,0 +1,18 @@ +# --- !Ups + +CREATE TABLE "cookie_authenticators" ( + "id" VARCHAR NOT NULL, + "provider_id" VARCHAR NOT NULL, + "provider_key" VARCHAR NOT NULL, + "last_used" TIMESTAMP NOT NULL, + "expiration" TIMESTAMP NOT NULL, + "idle_timeout" BIGINT NULL, + "cookie_max_age" BIGINT NULL, + "fingerprint" VARCHAR NULL +); + +CREATE INDEX cookie_authenticators_id ON cookie_authenticators (id); + +# --- !Downs + +DROP TABLE cookie_authenticators; \ No newline at end of file diff --git a/conf/logback.xml b/conf/logback.xml new file mode 100644 index 0000000..ddbdcfb --- /dev/null +++ b/conf/logback.xml @@ -0,0 +1,22 @@ + + + + + + + %coloredLevel - %logger - %message%n%xException + + + + + + + + + + + + diff --git a/conf/routes b/conf/routes new file mode 100644 index 0000000..20c89ec --- /dev/null +++ b/conf/routes @@ -0,0 +1,44 @@ +# Routes +# This file defines all application routes (Higher priority routes first) +# ~~~~ + +# Home page +GET / controllers.Application.index(versions: Map[String, Int] = Map()) +GET /versions controllers.Application.index(versions: Map[String, Int]) +GET /dependencies controllers.Application.dependencies(classified: Option[Boolean] = None, requiredTags: Seq[Int] ?= Seq(), noTag: Boolean ?= false) +GET /dependencies/classified controllers.Application.dependencies(classified: Option[Boolean] = Some(true), requiredTags: Seq[Int] ?= Seq(), noTag: Boolean ?= false) +GET /dependencies/unclassified controllers.Application.dependencies(classified: Option[Boolean] = Some(false), requiredTags: Seq[Int] ?= Seq(), noTag: Boolean ?= false) +POST /add-tag controllers.Application.addTag +POST /remove-tag controllers.Application.removeTag +POST /set-classified controllers.Application.setClassified(classified: Boolean) +POST /purge-cache controllers.Application.purgeCache(versions: Map[String, Int], next: String) +POST /snooze/:id controllers.Application.snooze(id: String, versions: Map[String, Int]) +POST /unsnooze/:snoozeId controllers.Application.unsnooze(snoozeId: Int, versions: Map[String, Int]) + +GET /https-test/with-redirect controllers.Application.testHttps(allowRedirect: Boolean = true) +GET /https-test controllers.Application.testHttps(allowRedirect: Boolean = false) + +GET /stats/basic controllers.Statistics.basic(selector: Option[String] = None) +GET /stats/basic/*selector controllers.Statistics.basic(selector: Option[String]) +GET /stats/details controllers.Statistics.vulnerabilities(selector: Option[String], tagId: Option[Int]) +GET /stats/libraries/vulnerable controllers.Statistics.vulnerableLibraries(selector: Option[String]) +GET /stats/libraries/all controllers.Statistics.allLibraries(selector: Option[String]) +GET /stats/libraries/gavs controllers.Statistics.allGavs(selector: Option[String]) + +GET /libraries/vulnerabilities controllers.Statistics.searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String]) + +GET /vulnerability/:name controllers.Statistics.vulnerability(name, selector: Option[String]) + +GET /tags.json controllers.Application.tagsExport + +GET /tags/import controllers.Application.tagsImport +POST /tags/import controllers.Application.tagsImportAction + +GET /routes.js controllers.Application.javascriptRoutes + +GET /sign-in controllers.AuthController.signIn +POST /sign-in controllers.AuthController.authenticate +POST /sign-out controllers.AuthController.signOut + +# Map static resources from the /public folder to the /assets URL path +GET /assets/*file controllers.Assets.versioned(path="/public", file: Asset) diff --git a/production.conf-example b/production.conf-example new file mode 100644 index 0000000..8ada3e4 --- /dev/null +++ b/production.conf-example @@ -0,0 +1,115 @@ +# This is the main configuration file for the application. +# ~~~~~ + +# Secret key +# ~~~~~ +# The secret key is used to secure cryptographics functions. +# +# This must be changed for production, but we recommend not changing it in this file. +# +# See https://www.playframework.com/documentation/latest/ApplicationSecret for more details. +play.crypto.secret = "{{ lookup('password', 'play_secret length=64') }}" + +# The application languages +# ~~~~~ +play.i18n.langs = [ "en" ] + +play.modules.enabled += "modules.ConfigModule" +play.modules.enabled += "modules.SilhouetteModule" + +app.hostname=… # You have to configure the hostname there. If you don't do so, all accesses via hostname will be prohibited. This is a protection against DNS rebind attacks. + +yssdc{ + bamboo{ + url = … + } + reports { + provider = "bamboo" + bamboo{ + user = … + password = … + } + } + projects = {jobId:humanReadableName, …} + teams = […] + exclusions{ + missingGAV{ + bySha1 = [] + } + } + projectsToTeams = { + … + } + teamLeaders = { # all teams used here must be listed above + team: leader, + … + } +} + +# Router +# ~~~~~ +# Define the Router object to use for this application. +# This router will be looked up first when the application is starting up, +# so make sure this is the entry point. +# Furthermore, it's assumed your route file is named properly. +# So for an application router like `my.application.Router`, +# you may need to define a router file `conf/my.application.routes`. +# Default to Routes in the root package (and conf/routes) +# play.http.router = my.application.Routes + +# Database configuration +# ~~~~~ +# You can declare as many datasources as you want. +# By convention, the default datasource is named `default` +# + +slick.dbs.default { + # Connection to internal database. It must be PostgreSQL. + driver = "slick.driver.PostgresDriver$" + db{ + url = "jdbc:postgresql://localhost/odca" + user = … + password = … + } +} +slick.dbs.odc { + # Connection to ODC database. It should be MySQL/MariaDB. H2 DB is not supported. PostgreSQL might work if you get ODC working with it, Other databases might be supported in future. + driver = "slick.driver.MySQLDriver$" + db { + url = "jdbc:mysql://127.0.0.1/dependencycheck" + # Those credentials are default in ODC (but you might have changed them): + user = "dcuser" + password = "DC-Pass1337!" + } +} + +# Evolutions +# ~~~~~ +# You can disable evolutions if needed +# play.evolutions.enabled=false + +# You can disable evolutions for a specific datasource if necessary +# play.evolutions.db.default.enabled=false + + + +silhouette { + # Authenticator settings + authenticator.cookieName = "authenticator" + authenticator.cookiePath = "/" + authenticator.secureCookie=false # is ignored; overriden in app/controllers/AuthController.scala; But it must be present! + authenticator.httpOnlyCookie = true + authenticator.useFingerprinting = true + authenticator.authenticatorIdleTimeout = 12 hours + authenticator.authenticatorExpiry = 12 hours + + authenticator.rememberMe.cookieMaxAge = 30 days + authenticator.rememberMe.authenticatorIdleTimeout = 5 days + authenticator.rememberMe.authenticatorExpiry = 30 days + + credentialsVerificationService{ + type="external" # verifies credentials at the URL specified below + url="http://localhost:9050/" + } +} + diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 0000000..325b32c --- /dev/null +++ b/project/build.properties @@ -0,0 +1,4 @@ +#Activator-generated Properties +#Wed Jul 15 12:08:24 CEST 2015 +template.uuid=a91771f5-1745-4f51-b877-badeea610f64 +sbt.version=0.13.9 diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..a180aa3 --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,16 @@ +// The Play plugin +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.4") + +// web plugins + +addSbtPlugin("com.typesafe.sbt" % "sbt-coffeescript" % "1.0.0") + +addSbtPlugin("com.typesafe.sbt" % "sbt-less" % "1.0.6") + +addSbtPlugin("com.typesafe.sbt" % "sbt-jshint" % "1.0.3") + +addSbtPlugin("com.typesafe.sbt" % "sbt-rjs" % "1.0.7") + +addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.0") + +addSbtPlugin("com.typesafe.sbt" % "sbt-mocha" % "1.1.0")