diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f701c0802..9960233e2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,8 @@ next release --------------------- - WARNING: Vulnerablecode V1 API and UI has stopped supporting Ubuntu OVAL advisories, please shift to V3 API for new Ubuntu advisories. +- WARNING: We will deprecate improver pipelines for calculating package version rank, grouping advisories for packages and calculating risk scores in the next release, we are doing it at advisory import time instead of as separate pipelines, this will improve the performance and consistency of the data. +- Calculate package verion rank, group advisories for packages and package risk score and advisory risk score during import of advisories. - Add attribute ``pipeline_id`` to AdvisoryV2 to track the pipeline that created the advisory, also rename existing ``datasource_id`` and AVIDs. Version v38.6.0 diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 634476ddf..973db8132 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -23,6 +23,7 @@ from rest_framework.reverse import reverse from rest_framework.throttling import AnonRateThrottle +from vulnerabilities.models import SSVC from vulnerabilities.models import AdvisoryAlias from vulnerabilities.models import AdvisoryReference from vulnerabilities.models import AdvisorySet @@ -30,13 +31,11 @@ from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import AdvisoryWeakness -from vulnerabilities.models import Group from vulnerabilities.models import GroupedAdvisory from vulnerabilities.models import ImpactedPackageAffecting from vulnerabilities.models import PackageV2 from vulnerabilities.throttling import PermissionBasedUserRateThrottle from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS -from vulnerabilities.utils import get_advisories_from_groups from vulnerabilities.utils import merge_and_save_grouped_advisories @@ -48,6 +47,7 @@ class PackageQuerySerializer(serializers.Serializer): ) details = serializers.BooleanField(default=False) ignore_qualifiers_subpath = serializers.BooleanField(default=False) + max_advisories = serializers.IntegerField(default=100, min_value=1, max_value=10000) def validate(self, data): if not data["purls"]: @@ -227,13 +227,25 @@ def get_affected_by_vulnerabilities(self, package): result = [] for adv in advisories: - fixed = impact_map.get(adv["avid"]) - adv.pop("avid", None) + fixed = impact_map.get(adv["advisory_uid"]) or [] + resource_url = None + + if request := self.context.get("request", None): + resource_url = adv.get("resource_url") or None + resource_url = request.build_absolute_uri(location=resource_url) result.append( { - **adv, + "advisory_id": adv["advisory_id"], + "advisory_uid": adv["advisory_uid"], + "aliases": adv["aliases"], + "summary": adv["summary"], + "weighted_severity": adv["weighted_severity"], + "exploitability": adv["exploitability"], + "risk_score": adv["risk_score"], + "ssvc_trees": adv["ssvc_trees"], "fixed_by_packages": fixed, + "resource_url": resource_url, } ) @@ -247,9 +259,20 @@ def get_affected_by_vulnerabilities(self, package): advisories_ids = advisories_qs.only("id") advisories_ids = list(advisories_ids[:101]) - if len(advisories_ids) > 100: + if len(advisories_ids) > self.context.get("max_advisories", 100): return None + advisories_qs = advisories_qs.prefetch_related( + "aliases", + Prefetch( + "related_ssvcs", + queryset=SSVC.objects.select_related("source_advisory") + .only("id", "decision", "options", "vector", "source_advisory__url") + .distinct("source_advisory__url"), + to_attr="prefetched_ssvc_trees", + ), + ) + advisory_by_avid = {adv.avid: adv for adv in advisories_qs} avids = advisory_by_avid.keys() @@ -265,8 +288,14 @@ def get_affected_by_vulnerabilities(self, package): for advisory in advisories_qs: impact = impact_by_avid.get(advisory.avid) - if not impact: - continue + fixed_by_packages = [] + if impact: + fixed_by_packages = [pkg.purl for pkg in impact.fixed_by_packages.all()] + + resource_url = None + + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri(location=advisory.get_absolute_url()) result.append( { @@ -274,73 +303,105 @@ def get_affected_by_vulnerabilities(self, package): "advisory_uid": advisory.avid, "aliases": [alias.alias for alias in advisory.aliases.all()], "summary": advisory.summary, - "severity": advisory.weighted_severity, + "weighted_severity": advisory.weighted_severity, "exploitability": advisory.exploitability, "risk_score": advisory.risk_score, - "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], + "fixed_by_packages": fixed_by_packages, + "resource_url": resource_url, + "ssvc_trees": [ + { + "vector": ssvc.vector, + "decision": ssvc.decision, + "options": ssvc.options, + "source_url": ssvc.source_advisory.url, + } + for ssvc in advisory.prefetched_ssvc_trees + ], } ) return result - if not advisories: - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - advisories_qs = advisories_qs.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, advisories_qs, "affecting" - ) - return self.return_advisories_data(package, advisories_qs, advisories) + # if not advisories: + # if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + # advisories_qs = advisories_qs.prefetch_related( + # "aliases", + # "impacted_packages__affecting_packages", + # "impacted_packages__fixed_by_packages", + # ) + # advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( + # package, advisories_qs, "affecting" + # ) + # return self.return_advisories_data(package, advisories_qs, advisories) def get_fixing_vulnerabilities(self, package): advisories = self.context["fixing_advisory_map"].get(package.id, []) - if advisories: - return advisories + results = [] + resource_url = None + for advisory in advisories: + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri(location=advisory["resource_url"]) + results.append( + { + "advisory_id": advisory["advisory_id"], + "resource_url": resource_url, + "advisory_uid": advisory["advisory_uid"], + } + ) + if results: + return results advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) - if not package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: advisories_ids = advisories_qs.only("id") advisories_ids = list(advisories_ids[:101]) - if len(advisories_ids) > 100: + if len(advisories_ids) > self.context.get("max_advisories", 100): return None results = [] for advisory in advisories_qs: + resource_url = None + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri(location=advisory.get_absolute_url()) results.append( { "advisory_id": advisory.advisory_id.split("/")[-1], "advisory_uid": advisory.avid, + "resource_url": resource_url, } ) return results - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - advisories_qs = advisories_qs.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - if not advisories_qs.exists(): - return [] - advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, advisories_qs, "fixing" - ) - return self.return_fixing_advisories_data(advisories) + # if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + # advisories_qs = advisories_qs.prefetch_related( + # "aliases", + # "impacted_packages__affecting_packages", + # "impacted_packages__fixed_by_packages", + # ) + # if not advisories_qs.exists(): + # return [] + # advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( + # package, advisories_qs, "fixing" + # ) + # return self.return_fixing_advisories_data(advisories) def return_fixing_advisories_data(self, advisories): result = [] for advisory in advisories: assert isinstance(advisory, GroupedAdvisory) + resource_url = None + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri( + location=advisory.advisory.get_absolute_url() + ) result.append( { "advisory_id": advisory.identifier, "advisory_uid": advisory.advisory.avid, + "resource_url": resource_url, } ) @@ -361,22 +422,28 @@ def return_advisories_data(self, package, advisories_qs, advisories): result = [] for advisory in advisories: assert isinstance(advisory, GroupedAdvisory) + resource_url = None + fixed_by_packages = [] + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri( + location=advisory.advisory.get_absolute_url() + ) impact = impact_by_avid.get(advisory.advisory.avid) - if not impact: - continue + if impact: + fixed_by_packages = list(set([pkg.purl for pkg in impact.fixed_by_packages.all()])) result.append( { "advisory_id": advisory.identifier, "advisory_uid": advisory.advisory.avid, "aliases": [alias.alias for alias in advisory.aliases], + "summary": advisory.advisory.summary, "weighted_severity": advisory.weighted_severity, "exploitability": advisory.exploitability, "risk_score": advisory.risk_score, - "summary": advisory.advisory.summary, - "fixed_by_packages": list( - set([pkg.purl for pkg in impact.fixed_by_packages.all()]) - ), + "fixed_by_packages": fixed_by_packages, + "resource_url": resource_url, + "ssvc_trees": advisory.ssvc_trees, } ) @@ -405,6 +472,7 @@ def create(self, request, *args, **kwargs): purls = serializer.validated_data["purls"] details = serializer.validated_data["details"] ignore_qualifiers_subpath = serializer.validated_data["ignore_qualifiers_subpath"] + max_advisories = serializer.validated_data["max_advisories"] if not purls: impacted = ImpactedPackageAffecting.objects.filter(package_id=OuterRef("id")) @@ -469,6 +537,7 @@ def create(self, request, *args, **kwargs): "advisory_map": affected_advisory_map, "impact_map": impact_map, "fixing_advisory_map": fixing_advisory_map, + "max_advisories": max_advisories, }, ) return self.get_paginated_response(serializer.data) @@ -583,7 +652,27 @@ def get_affected_advisories_bulk(packages): relation_type="affecting", ) .select_related("primary_advisory") - .prefetch_related(Prefetch("aliases", queryset=AdvisoryAlias.objects.only("alias"))) + .prefetch_related( + Prefetch("aliases", queryset=AdvisoryAlias.objects.only("alias")), + Prefetch( + "members", + queryset=AdvisorySetMember.objects.select_related("advisory").prefetch_related( + Prefetch( + "advisory__related_ssvcs", + queryset=SSVC.objects.select_related("source_advisory") + .only( + "id", + "options", + "decision", + "vector", + "source_advisory__url", + ) + .distinct("source_advisory__url"), + to_attr="prefetched_ssvc_trees", + ) + ), + ), + ) .annotate( max_severity=Max( "members__advisory__weighted_severity", @@ -627,6 +716,22 @@ def get_affected_advisories_bulk(packages): identifier = primary.advisory_id.split("/")[-1] aliases = [a for a in adv._aliases_cache if a != identifier] + all_ssvc = [] + + for member in adv.members.all(): + all_ssvc.extend(member.advisory.prefetched_ssvc_trees) + + ssvcs = [] + + for ssvc in all_ssvc: + ssvcs.append( + { + "vector": ssvc.vector, + "decision": ssvc.decision, + "options": ssvc.options, + "source_url": ssvc.source_advisory.url, + } + ) grouped.append( { @@ -637,6 +742,8 @@ def get_affected_advisories_bulk(packages): "exploitability": exploitability, "risk_score": risk_score, "summary": primary.summary, + "resource_url": primary.get_absolute_url(), + "ssvc_trees": ssvcs, } ) @@ -697,7 +804,7 @@ def get_fixing_advisories_bulk(packages): package_map = defaultdict(list) for adv in advisory_sets: - package_map[adv.package_id].append(adv.primary_advisory.advisory_id) + package_map[adv.package_id].append(adv.primary_advisory) result = {} @@ -705,9 +812,13 @@ def get_fixing_advisories_bulk(packages): groups = package_map.get(package.id, []) grouped = [] - for adv_id in groups: + for advisory in groups: grouped.append( - {"advisory_id": adv_id.split("/")[-1], "advisory_uid": adv_id.split("/")[-1]} + { + "advisory_id": advisory.advisory_id.split("/")[-1], + "resource_url": advisory.get_absolute_url(), + "advisory_uid": advisory.avid, + } ) result[package.id] = grouped diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 5b47a7cf1..b1db301d4 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -11,7 +11,6 @@ from vulnerabilities.improvers import vulnerability_status from vulnerabilities.pipelines import add_cvss31_to_CVEs from vulnerabilities.pipelines import compute_package_risk -from vulnerabilities.pipelines import compute_package_version_rank from vulnerabilities.pipelines import enhance_with_exploitdb from vulnerabilities.pipelines import enhance_with_kev from vulnerabilities.pipelines import enhance_with_metasploit @@ -32,7 +31,6 @@ enhance_with_metasploit as enhance_with_metasploit_v2, ) from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2 -from vulnerabilities.pipelines.v2_improvers import group_advisories_for_packages from vulnerabilities.pipelines.v2_improvers import reference_collect_commits from vulnerabilities.pipelines.v2_improvers import relate_severities from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2 @@ -61,7 +59,6 @@ enhance_with_metasploit.MetasploitImproverPipeline, enhance_with_exploitdb.ExploitDBImproverPipeline, compute_package_risk.ComputePackageRiskPipeline, - compute_package_version_rank.ComputeVersionRankPipeline, add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline, remove_duplicate_advisories.RemoveDuplicateAdvisoriesPipeline, populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline, @@ -75,7 +72,6 @@ collect_ssvc_trees.CollectSSVCPipeline, relate_severities.RelateSeveritiesPipeline, archive_urls.ArchiveImproverPipeline, - group_advisories_for_packages.GroupAdvisoriesForPackages, compute_advisory_todo_v2.ComputeToDo, reference_collect_commits.CollectReferencesFixCommitsPipeline, enhance_with_github_poc.GithubPocsImproverPipeline, diff --git a/vulnerabilities/migrations/0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more.py b/vulnerabilities/migrations/0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more.py new file mode 100644 index 000000000..8f45487b2 --- /dev/null +++ b/vulnerabilities/migrations/0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 5.2.11 on 2026-05-26 08:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0132_migrate_advisoryv2_datasource_ids"), + ] + + operations = [ + migrations.AlterField( + model_name="advisoryv2", + name="advisory_id", + field=models.CharField( + db_index=True, + help_text="An advisory is a unique vulnerability identifier in some database, such as PYSEC-2020-2233", + max_length=200, + ), + ), + migrations.AlterField( + model_name="advisoryv2", + name="avid", + field=models.CharField( + help_text="Unique ID for the datasource used for this advisory .e.g.: pysec_importer_v2/PYSEC-2020-2233", + max_length=250, + ), + ), + migrations.AlterField( + model_name="advisoryv2", + name="datasource_id", + field=models.CharField( + db_index=True, + help_text="Unique ID for the datasource used for this advisory .e.g.: nginx", + max_length=50, + ), + ), + migrations.AlterField( + model_name="advisoryv2", + name="pipeline_id", + field=models.CharField( + db_index=True, + help_text="Unique ID for the pipeline used for this advisory .e.g.: nginx_importer_v2", + max_length=50, + ), + ), + ] diff --git a/vulnerabilities/migrations/0134_alter_advisoryset_unique_together.py b/vulnerabilities/migrations/0134_alter_advisoryset_unique_together.py new file mode 100644 index 000000000..7b0c2a4be --- /dev/null +++ b/vulnerabilities/migrations/0134_alter_advisoryset_unique_together.py @@ -0,0 +1,17 @@ +# Generated by Django 5.2.11 on 2026-05-28 13:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more"), + ] + + operations = [ + migrations.AlterUniqueTogether( + name="advisoryset", + unique_together={("package", "relation_type", "primary_advisory")}, + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 69253f54b..8ed6d2520 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -19,6 +19,7 @@ from itertools import groupby from operator import attrgetter from traceback import format_exc as traceback_format_exc +from typing import Dict from typing import List from typing import NamedTuple from typing import Optional @@ -3054,7 +3055,7 @@ class AdvisoryV2(models.Model): # This is similar to a type or a namespace datasource_id = models.CharField( - max_length=200, + max_length=50, blank=False, null=False, db_index=True, @@ -3062,7 +3063,7 @@ class AdvisoryV2(models.Model): ) pipeline_id = models.CharField( - max_length=200, + max_length=50, blank=False, null=False, db_index=True, @@ -3071,7 +3072,7 @@ class AdvisoryV2(models.Model): # This is similar to a name advisory_id = models.CharField( - max_length=500, + max_length=200, blank=False, null=False, unique=False, @@ -3081,7 +3082,7 @@ class AdvisoryV2(models.Model): ) avid = models.CharField( - max_length=500, + max_length=250, blank=False, null=False, help_text="Unique ID for the datasource used for this advisory ." @@ -3872,6 +3873,7 @@ class GroupedAdvisory(NamedTuple): weighted_severity: Optional[float] exploitability: Optional[float] risk_score: Optional[float] + ssvc_trees: List[Dict] class AdvisoryPOC(models.Model): diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index ea6fc9185..642b5e3d2 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -7,21 +7,17 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from typing import List - -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import Group from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set +from vulnerabilities.pipes.group_advisories import group_advisory_for_package from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS -from vulnerabilities.utils import merge_advisories class GroupAdvisoriesForPackages(VulnerableCodePipeline): """Group advisories for packages that have multiple importers""" pipeline_id = "group_advisories_for_packages" + run_once = True @classmethod def steps(cls): @@ -33,28 +29,4 @@ def group_advisories_for_packages(self): def group_advisoris_for_packages(logger=None): for package in PackageV2.objects.filter(type__in=TYPES_WITH_MULTIPLE_IMPORTERS).iterator(): - logger(f"Grouping advisories for package {package.purl}") - affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( - purl=package.purl - ).prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( - purl=package.purl - ).prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - try: - affected_groups: List[Group] = merge_advisories(affecting_advisories, package) - fixed_by_groups: List[Group] = merge_advisories(fixed_by_advisories, package) - delete_and_save_advisory_set(affected_groups, package, relation="affecting") - delete_and_save_advisory_set(fixed_by_groups, package, relation="fixing") - except Exception as e: - logger(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") - continue + group_advisory_for_package(package, logger=logger) diff --git a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py index 9d874c635..6e8831906 100644 --- a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py +++ b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py @@ -12,6 +12,7 @@ from traceback import format_exc as traceback_format_exc from aboutcode.pipeline import LoopProgress +from django.db import transaction from django.db.models import F from django.db.models import Q from django.utils import timezone @@ -26,6 +27,8 @@ from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipes.fetchcode_utils import get_versions +from vulnerabilities.pipes.group_advisories import group_advisory_for_package +from vulnerabilities.pipes.risk_score import compute_package_risk_score from vulnerabilities.utils import update_purl_version @@ -159,6 +162,7 @@ def get_purl_versions(purl, cached_versions, logger): return cached_versions.get(purl) or [] +@transaction.atomic def bulk_create_with_m2m(purls, impact, relation, logger): """Bulk create PackageV2 and also bulk populate M2M Impact and Package relationships.""" if not purls: @@ -181,6 +185,13 @@ def bulk_create_with_m2m(purls, impact, relation, logger): logger(f"Error creating ImpactedPackage {relation}: {e!r} \n {traceback_format_exc()}") return 0 + for pkg in affected_packages_v2: + group_advisory_for_package(pkg, logger=logger) + risk_score = compute_package_risk_score(pkg) + logger(f"Computed risk score {risk_score} for package {pkg.purl}") + pkg.risk_score = risk_score + pkg.save() + return len(relations) diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 732e2e0ab..21ca7b996 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -47,6 +47,9 @@ from vulnerabilities.models import VulnerabilityRelatedReference from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness +from vulnerabilities.pipes.group_advisories import group_advisory_for_package +from vulnerabilities.pipes.risk_score import compute_advisory_risk_score +from vulnerabilities.pipes.risk_score import compute_package_risk_score from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 @@ -362,6 +365,12 @@ def insert_advisory_v2( if values: getattr(advisory_obj, field_name).add(*values) + weighted_severity, exploitability, risk_score = compute_advisory_risk_score(advisory_obj) + advisory_obj.weighted_severity = round(weighted_severity, 1) if weighted_severity is not None else None + advisory_obj.exploitability = round(exploitability, 1) if exploitability is not None else None + advisory_obj.risk_score = round(risk_score, 1) if risk_score is not None else None + advisory_obj.save() + for affected_pkg in advisory.affected_packages: impact = ImpactedPackage.objects.create( advisory=advisory_obj, @@ -390,6 +399,48 @@ def insert_advisory_v2( impact.affecting_packages.add(*affected_packages_v2) impact.fixed_by_packages.add(*fixed_packages_v2) + for pkg in list(affected_packages_v2): + logger(f"Grouping advisories for package: {pkg.purl}") + try: + group_advisory_for_package( + pkg, + logger=logger, + current_advisory=advisory_obj, + current_advisory_relation="affecting", + ) + except Exception as e: + logger( + f"Failed to group advisories for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + try: + risk_score = compute_package_risk_score(pkg, current_advisory_risk_score=advisory_obj.risk_score) + logger(f"Computed risk score {risk_score} for package {pkg.purl}") + pkg.risk_score = risk_score + pkg.save() + pkg.calculate_version_rank + except Exception as e: + logger( + f"Failed to compute risk score for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + + for pkg in list(fixed_packages_v2): + logger(f"Grouping advisories for package: {pkg.purl}") + try: + group_advisory_for_package( + pkg, + logger=logger, + current_advisory=advisory_obj, + current_advisory_relation="fixing", + ) + except Exception as e: + logger( + f"Failed to group advisories for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + pkg.calculate_version_rank + introduced_commit_v2 = get_or_create_advisory_package_commit_patches( affected_pkg.introduced_by_commit_patches ) diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py index 983ac3386..0f8981e45 100644 --- a/vulnerabilities/pipes/group_advisories.py +++ b/vulnerabilities/pipes/group_advisories.py @@ -7,14 +7,19 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from collections import defaultdict +from typing import List + from django.db import transaction +from vulnerabilities.models import AdvisorySet +from vulnerabilities.models import AdvisorySetMember +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import Group + @transaction.atomic def delete_and_save_advisory_set(groups, package, relation=None): - from vulnerabilities.models import AdvisorySet - from vulnerabilities.models import AdvisorySetMember - from vulnerabilities.models import Group AdvisorySet.objects.filter(package=package, relation_type=relation).delete() @@ -50,3 +55,71 @@ def delete_and_save_advisory_set(groups, package, relation=None): ) AdvisorySetMember.objects.bulk_create(membership_to_create) + + +def group_advisory_for_package( + package, logger=None, current_advisory=None, current_advisory_relation=None +): + """ + Group advisories for a given package and save the advisory sets for the package. + """ + from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS + + if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: + return + + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + + fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + + if current_advisory and current_advisory_relation == "affecting": + affecting_advisories = list(affecting_advisories) + [current_advisory] + elif current_advisory and current_advisory_relation == "fixing": + fixed_by_advisories = list(fixed_by_advisories) + [current_advisory] + + try: + affected_groups: List[Group] = merge_advisories(affecting_advisories, package) + fixed_by_groups: List[Group] = merge_advisories(fixed_by_advisories, package) + delete_and_save_advisory_set(affected_groups, package, relation="affecting") + delete_and_save_advisory_set(fixed_by_groups, package, relation="fixing") + logger(f"Successfully rebuilt advisory sets for package {package.purl}") + except Exception as e: + if logger: + logger(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") + return + + +def merge_advisories(advisories, package): + """ + Merge advisories based on their content hash and identifiers. + """ + from vulnerabilities.utils import compute_advisory_content_hash + from vulnerabilities.utils import get_merged_identifier_groups + + advisories = list(advisories) + + content_hash_map = defaultdict(list) + + for adv in advisories: + content_hash = compute_advisory_content_hash(adv, package) + content_hash_map[content_hash].append(adv) + + final_groups: List[Group] = [] + + for group in content_hash_map.values(): + groups = get_merged_identifier_groups(group) + final_groups.extend(groups) + + return final_groups diff --git a/vulnerabilities/pipes/risk_score.py b/vulnerabilities/pipes/risk_score.py new file mode 100644 index 000000000..24cc19a5b --- /dev/null +++ b/vulnerabilities/pipes/risk_score.py @@ -0,0 +1,77 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.db.models import Max + +from vulnerabilities.models import AdvisoryV2 + +from decimal import Decimal, ROUND_HALF_UP + +def quantize_1(value): + if value is None: + return None + + return Decimal(str(value)).quantize( + Decimal("0.1"), + rounding=ROUND_HALF_UP, + ) + + +def compute_package_risk_score(package, current_advisory_risk_score=None): + """Calculate the risk score for a single PackageV2 object.""" + max_risk = ( + AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) + .aggregate(max_risk=Max("risk_score")) + .get("max_risk") + ) + # include current advisory risk score in the calculation if provided and is higher than the max risk score from the database + if current_advisory_risk_score is not None: + max_risk = max(max_risk or 0, current_advisory_risk_score) + if max_risk is None: + return None + return round(float(max_risk), 1) + + +def compute_advisory_risk_score(advisory): + """ + Calculate the risk score for a single AdvisoryV2 object. + Returns a tuple of (weighted_severity, exploitability, risk_score). + """ + from vulnerabilities.risk import compute_vulnerability_risk_factors + + weighted_severity = None + exploitability = None + risk_score = None + + references = advisory.references.all() + exploits = advisory.exploits.all() + + severities = list(advisory.severities.all()) + + for rel in advisory.related_advisory_severities.all(): + severities.extend(rel.severities.all()) + + try: + calculated_weighted_severity, calculated_exploitability = ( + compute_vulnerability_risk_factors( + references=references, + severities=severities, + exploits=exploits, + ) + ) + + weighted_severity = calculated_weighted_severity + exploitability = calculated_exploitability + if exploitability and weighted_severity: + risk_score = min(float(exploitability * weighted_severity), 10.0) + risk_score = round(risk_score, 1) + except Exception as e: + risk_score = None + + return quantize_1(weighted_severity), quantize_1(exploitability), quantize_1(risk_score) diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 2e618a920..0b313974c 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -35,6 +35,7 @@ import urllib3 from cwe2.database import Database from cwe2.database import InvalidCWEError +from django.db.models import Prefetch from packageurl import PackageURL from packageurl.contrib.django.utils import without_empty_values from univers.version_range import RANGE_CLASS_BY_SCHEMES @@ -43,7 +44,6 @@ from univers.version_range import VersionRange from aboutcode.hashid import build_vcid -from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set logger = logging.getLogger(__name__) @@ -868,29 +868,6 @@ def compute_patch_checksum(patch_text: str): return hashlib.sha512(patch_text.encode("utf-8")).hexdigest() -def merge_advisories(advisories, package): - """ - Merge advisories based on their content hash and identifiers. - """ - from vulnerabilities.models import Group - - advisories = list(advisories) - - content_hash_map = defaultdict(list) - - for adv in advisories: - content_hash = compute_advisory_content_hash(adv, package) - content_hash_map[content_hash].append(adv) - - final_groups: List[Group] = [] - - for group in content_hash_map.values(): - groups = get_merged_identifier_groups(group) - final_groups.extend(groups) - - return final_groups - - def compute_advisory_content_hash(adv, package): """Compute a content hash for an advisory based on its affected and fixed packages for a given package. This is used to determine if two advisories are the same based on their content.""" @@ -906,8 +883,12 @@ def compute_advisory_content_hash(adv, package): ) for impact in adv.impacted_packages.filter(base_purl=str(version_less_purl)): - affected.extend([pkg.package_url for pkg in impact.affecting_packages.all()]) - fixed.extend([pkg.package_url for pkg in impact.fixed_by_packages.all()]) + for pkg in impact.affecting_packages.all(): + if pkg.package_url: + affected.extend([pkg.package_url]) + for pkg in impact.fixed_by_packages.all(): + if pkg.package_url: + fixed.extend([pkg.package_url]) normalized_data = { "affected_packages": normalize_list(affected), @@ -979,10 +960,12 @@ def get_merged_identifier_groups(advisories): return final_groups -def get_advisories_from_groups(groups): +def get_advisories_from_groups(groups, include_ssvc_trees=False): """ Return a list of advisories from the merged groups of advisories. """ + from vulnerabilities.models import SSVC + from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import Group from vulnerabilities.models import GroupedAdvisory @@ -1016,6 +999,35 @@ def get_advisories_from_groups(groups): identifier = group.primary.advisory_id.split("/")[-1] filtered_aliases = [alias for alias in group.aliases if alias.alias != identifier] + ssvc_trees = [] + + if include_ssvc_trees: + + all_advs = [group.primary] + list(group.secondaries) + + advisories_qs = AdvisoryV2.objects.filter( + id__in=[adv.id for adv in all_advs] + ).prefetch_related( + Prefetch( + "related_ssvcs", + queryset=SSVC.objects.select_related("source_advisory") + .only("id", "vector", "decision", "options", "source_advisory__url") + .distinct(), + to_attr="ssvc_trees", + ) + ) + + ssvc_trees = [ + { + "vector": ssvc.vector, + "decision": ssvc.decision, + "options": ssvc.options, + "url": ssvc.source_advisory.url if ssvc.source_advisory else None, + } + for adv in advisories_qs + for ssvc in adv.ssvc_trees + ] + advisories.append( GroupedAdvisory( aliases=filtered_aliases, @@ -1024,6 +1036,7 @@ def get_advisories_from_groups(groups): weighted_severity=weighted_severity, exploitability=exploitability, risk_score=risk_score, + ssvc_trees=ssvc_trees or [], ) ) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 3aff06768..d7a23ffab 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -390,41 +390,41 @@ def get_context_data(self, **kwargs): return context - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( - purl=package.purl - ) - - fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( - purl=package.purl - ) - fixed_pkg_details = get_fixed_package_details(package) - context["fixed_package_details"] = fixed_pkg_details - context["grouped"] = True - - affecting_advisories = affecting_advisories.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - affected_by_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, affecting_advisories, "affecting" - ) - - fixed_by_advisories = fixed_by_advisories.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - fixing_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, fixed_by_advisories, "fixing" - ) - - context["affected_by_advisories_v2"] = affected_by_advisories - context["fixing_advisories_v2"] = fixing_advisories - return context + # if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + # affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + # purl=package.purl + # ) + + # fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + # purl=package.purl + # ) + # fixed_pkg_details = get_fixed_package_details(package) + # context["fixed_package_details"] = fixed_pkg_details + # context["grouped"] = True + + # affecting_advisories = affecting_advisories.prefetch_related( + # "aliases", + # "impacted_packages__affecting_packages", + # "impacted_packages__fixed_by_packages", + # ) + + # affected_by_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( + # package, affecting_advisories, "affecting" + # ) + + # fixed_by_advisories = fixed_by_advisories.prefetch_related( + # "aliases", + # "impacted_packages__affecting_packages", + # "impacted_packages__fixed_by_packages", + # ) + + # fixing_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( + # package, fixed_by_advisories, "fixing" + # ) + + # context["affected_by_advisories_v2"] = affected_by_advisories + # context["fixing_advisories_v2"] = fixing_advisories + # return context def get_object(self, queryset=None): if queryset is None: