Handle vulnerabilities which don't have any vulnerability ids

requirements.txt

@@ -55,3 +55,4 @@ zipp==0.6.0
 requests==2.23.0
 toml==0.10.2
 PyYAML==5.3.1
+freezegun==1.1.0

vulnerabilities/api.py

@@ -63,7 +63,6 @@ class Meta:
 
 
 class HyperLinkedVulnerabilitySerializer(serializers.HyperlinkedModelSerializer):
- vulnerability_id = serializers.CharField(source="cve_id")
 
  class Meta:
  model = Vulnerability
@@ -177,7 +176,6 @@ def bulk_search(self, request):
 
 
 class VulnerabilityFilterSet(filters.FilterSet):
- vulnerability_id = filters.CharFilter(field_name="cve_id")
 
  class Meta:
  model = Vulnerability
@@ -208,13 +206,13 @@ def bulk_search(self, request):
  },
  )
 
- for cve_id in request.data["vulnerabilities"]:
- filter_list.append(cve_id)
+ for vulnerability_id in request.data["vulnerabilities"]:
+ filter_list.append(vulnerability_id)
  # This handles the case when the said cve doesnt exist in db
- response[cve_id] = {}
- res = Vulnerability.objects.filter(cve_id__in=filter_list)
+ response[vulnerability_id] = {}
+ res = Vulnerability.objects.filter(vulnerability_id__in=filter_list)
  for vuln in res:
- response[vuln.cve_id] = MinimalVulnerabilitySerializer(
+ response[vuln.vulnerability_id] = MinimalVulnerabilitySerializer(
  vuln, context={"request": request}
  ).data
  return Response(response)

vulnerabilities/data_source.py

@@ -81,14 +81,14 @@ class Advisory:
  impacted_package_urls: Iterable[PackageURL]
  resolved_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list)
  vuln_references: List[Reference] = dataclasses.field(default_factory=list)
-cve_id: Optional[str] = None
+vulnerability_id: Optional[str] = None
 
  def __hash__(self):
  s = "{}{}{}{}".format(
  self.summary,
-"".join(sorted([str(p) for p in self.impacted_package_urls])),
-"".join(sorted([str(p) for p in self.resolved_package_urls])),
- self.cve_id,
+''.join(sorted([str(p) for p in self.impacted_package_urls])),
+''.join(sorted([str(p) for p in self.resolved_package_urls])),
+ self.vulnerability_id,
  )
  return hash(s)
 
@@ -539,8 +539,6 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
  summary=description,
  impacted_package_urls=affected_purls,
  resolved_package_urls=safe_purls,
- cve_id=vuln_id,
- vuln_references=references,
- )
- )
+ vulnerability_id=vuln_id,
+ vuln_references=references))
  return all_adv

vulnerabilities/fixtures/debian.json

@@ -3,7 +3,7 @@
  "model": "vulnerabilities.vulnerability",
  "pk": 1,
  "fields": {
- "cve_id": "CVE-2014-8242",
+ "vulnerability_id": "CVE-2014-8242",
  "summary": ""
 
  }
@@ -12,7 +12,7 @@
  "model": "vulnerabilities.vulnerability",
  "pk": 2,
  "fields": {
- "cve_id": "CVE-2009-1382",
+ "vulnerability_id": "CVE-2009-1382",
  "summary": ""
 
  }
@@ -21,7 +21,7 @@
  "model": "vulnerabilities.vulnerability",
  "pk": 3,
  "fields": {
- "cve_id": "CVE-2009-2459",
+ "vulnerability_id": "CVE-2009-2459",
  "summary": ""
 
  }

vulnerabilities/import_runner.py

@@ -153,12 +153,23 @@ def process_advisories(data_source: DataSource) -> None:
  existing_ref = get_vuln_pkg_refs(vuln, pkg)
  if not existing_ref:
  bulk_create_vuln_pkg_refs.add(pkg_vuln_ref)
+ # A vulnerability-package relationship does not exist already
+ # if either the vulnerability or the package is just created.
 
  else:
- # This handles conflicts between existing data and obtained data
- if existing_ref[0].is_vulnerable != pkg_vuln_ref.is_vulnerable:
- handle_conflicts([existing_ref[0], pkg_vuln_ref.to_model_object()])
- existing_ref.delete()
+ # insert only if it there is no existing vulnerability-package relationship. # nopep8
+ existing_ref = get_vuln_pkg_refs(vuln, pkg)
+ if not existing_ref:
+ bulk_create_vuln_pkg_refs.add(pkg_vuln_ref)
+
+ else:
+ # This handles conflicts between existing data and obtained data
+ if existing_ref[0].is_vulnerable != pkg_vuln_ref.is_vulnerable:
+ handle_conflicts(
+ [existing_ref[0], pkg_vuln_ref.to_model_object()]
+ )
+ existing_ref.delete()
+
  except Exception:
  # TODO: store error but continue
  logger.error(
@@ -223,26 +234,13 @@ def _get_or_create_vulnerability(
  advisory: Advisory,
 ) -> Tuple[models.Vulnerability, bool]:
 
- if advisory.cve_id:
- query_kwargs = {"cve_id": advisory.cve_id}
- elif advisory.summary:
- query_kwargs = {"summary": advisory.summary}
- else:
- return models.Vulnerability.objects.create(), True
-
- try:
- vuln, created = models.Vulnerability.objects.get_or_create(**query_kwargs)
- # Eventually we only want to keep summary from NVD and ignore other descriptions.
- if advisory.summary and vuln.summary != advisory.summary:
- vuln.summary = advisory.summary
- vuln.save()
- return vuln, created
-
- except Exception:
- logger.error(
- f"Failed to _get_or_create_vulnerability: {query_kwargs!r}:\n" + traceback.format_exc()
- )
- raise
+ vuln, created = models.Vulnerability.objects.get_or_create(vulnerability_id=advisory.vulnerability_id) # nopep8
+ # Eventually we only want to keep summary from NVD and ignore other descriptions.
+ if advisory.summary and vuln.summary != advisory.summary:
+ vuln.summary = advisory.summary
+ vuln.save()
+
+ return vuln, created
 
 
 def _get_or_create_package(p: PackageURL) -> Tuple[models.Package, bool]:

vulnerabilities/importer_yielder.py

@@ -58,16 +58,16 @@
  'debian_tracker_url': 'https://security-tracker.debian.org/tracker/data/json'
  },
  },
-#  {
-#  'name': 'safetydb',
-#  'license': 'cc-by-nc-4.0',
-#  'last_run': None,
-#  'data_source': 'SafetyDbDataSource',
-#  'data_source_cfg': {
-#  'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json', # nopep8
-#  'etags': {}
-#  },
-#  },
+ {
+ 'name': 'safetydb',
+ 'license': 'cc-by-nc-4.0',
+ 'last_run': None,
+ 'data_source': 'SafetyDbDataSource',
+ 'data_source_cfg': {
+ 'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json', # nopep8
+ 'etags': {}
+ },
+ },
  {
  'name': 'npm',
  'license': 'mit',

vulnerabilities/importers/__init__.py

@@ -47,3 +47,4 @@
 from vulnerabilities.importers.suse_scores import SUSESeverityScoreDataSource
 from vulnerabilities.importers.ubuntu import UbuntuDataSource
 from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource
+from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource

vulnerabilities/importers/alpine_linux.py

@@ -187,7 +187,7 @@ def _load_advisories(
  impacted_package_urls=[],
  resolved_package_urls=resolved_purls,
  vuln_references=references,
-cve_id=vuln_ids[0].upper() if vuln_ids[0] != "CVE-????-?????" else None,
+vulnerability_id=vuln_ids[0] if vuln_ids[0] != "CVE-????-?????" else None,
  )
  )
 

vulnerabilities/importers/apache_kafka.py

@@ -82,7 +82,7 @@ def to_advisory(self, advisory_page):
 
  advisories.append(
  Advisory(
-cve_id=cve_id,
+vulnerability_id=cve_id,
  summary=cve_description_paragraph.text,
  impacted_package_urls=affected_packages,
  resolved_package_urls=fixed_packages,

vulnerabilities/importers/apache_tomcat.py

@@ -116,7 +116,7 @@ def to_advisories(self, apache_tomcat_advisory_html):
  summary="",
  impacted_package_urls=affected_packages,
  resolved_package_urls=fixed_package,
-cve_id=cve_id,
+vulnerability_id=cve_id,
  vuln_references=references,
  )
  )

vulnerabilities/importers/archlinux.py

@@ -130,7 +130,7 @@ def _parse(self, record) -> List[Advisory]:
 
  advisories.append(
  Advisory(
-cve_id=cve_id,
+vulnerability_id=cve_id,
  summary="",
  impacted_package_urls=impacted_purls,
  resolved_package_urls=resolved_purls,

vulnerabilities/importers/debian.py

@@ -151,7 +151,7 @@ def _parse(self, pkg_name: str, records: Mapping[str, Any]) -> List[Advisory]:
 
  advisories.append(
  Advisory(
-cve_id=cve_id,
+vulnerability_id=cve_id,
  summary=record.get("description", ""),
  impacted_package_urls=impacted_purls,
  resolved_package_urls=resolved_purls,

vulnerabilities/importers/elixir_security.py

@@ -139,6 +139,6 @@ def process_file(self, path):
  summary=yaml_file["description"],
  impacted_package_urls=vuln_purls,
  resolved_package_urls=safe_purls,
-cve_id=cve_id,
+vulnerability_id=cve_id,
  vuln_references=vuln_references,
  )

vulnerabilities/importers/gentoo.py

@@ -77,7 +77,7 @@ def process_file(self, file):
  # this way, but there seems no alternative.
  for cve in xml_data["cves"]:
  advisory = Advisory(
-cve_id=cve,
+vulnerability_id=cve,
  summary=xml_data["description"],
  impacted_package_urls=xml_data["affected_purls"],
  resolved_package_urls=xml_data["unaffected_purls"],

vulnerabilities/importers/github.py

@@ -229,7 +229,7 @@ def process_response(self) -> List[Advisory]:
  for cve_id in cve_ids:
  adv_list.append(
  Advisory(
-cve_id=cve_id,
+vulnerability_id=cve_id,
  summary=vuln_desc,
  impacted_package_urls=affected_purls,
  resolved_package_urls=unaffected_purls,

vulnerabilities/importers/kaybee.py

@@ -66,7 +66,7 @@ def yaml_file_to_advisory(yaml_path):
  references.append(Reference(url=f"{commit['repository']}/{commit['id']}"))
 
  return Advisory(
-cve_id=vuln_id,
+vulnerability_id=vuln_id,
  summary=summary,
  impacted_package_urls=impacted_packages,
  resolved_package_urls=resolved_packages,

vulnerabilities/importers/nginx.py

@@ -109,7 +109,7 @@ def to_advisories(self, data):
 
  advisories.append(
  Advisory(
-cve_id=cve_id,
+vulnerability_id=cve_id,
  summary=summary,
  impacted_package_urls=vulnerable_packages,
  resolved_package_urls=fixed_packages,

vulnerabilities/importers/npm.py

@@ -106,7 +106,7 @@ def process_file(self, file) -> List[Advisory]:
  advisories.append(
  Advisory(
  summary=record.get("overview", ""),
-cve_id=cve_id,
+vulnerability_id=cve_id,
  impacted_package_urls=impacted_purls,
  resolved_package_urls=resolved_purls,
  vuln_references=vuln_reference,

vulnerabilities/importers/nvd.py

@@ -89,7 +89,7 @@ def to_advisories(self, nvd_data):
  )
  summary = self.extract_summary(cve_item)
  yield Advisory(
-cve_id=cve_id, summary=summary, vuln_references=references, impacted_package_urls=[]
+vulnerability_id=cve_id, summary=summary, vuln_references=references, impacted_package_urls=[] # nopep8
  )
 
  @staticmethod

vulnerabilities/importers/openssl.py

@@ -111,7 +111,7 @@ def to_advisories(xml_response: str) -> Set[Advisory]:
  }
 
  advisory = Advisory(
-cve_id=cve_id,
+vulnerability_id=cve_id,
  summary=summary,
  impacted_package_urls=vuln_purls,
  resolved_package_urls=safe_purls,

vulnerabilities/importers/postgresql.py

@@ -105,7 +105,7 @@ def to_advisories(data):
 
  advisories.append(
  Advisory(
-cve_id=cve_id,
+vulnerability_id=cve_id,
  summary=summary,
  vuln_references=references,
  impacted_package_urls=affected_packages,

vulnerabilities/importers/redhat.py

@@ -138,7 +138,7 @@ def to_advisory(advisory_data):
 
  return Advisory(
  summary=advisory_data["bugzilla_description"],
-cve_id=advisory_data["CVE"],
+vulnerability_id=advisory_data["CVE"],
  impacted_package_urls=affected_purls,
  vuln_references=references,
  )

vulnerabilities/importers/retiredotnet.py

@@ -99,5 +99,5 @@ def process_file(self, path) -> List[Advisory]:
  summary=json_doc['description'],
  impacted_package_urls=affected_purls,
  resolved_package_urls=fixed_purls,
-cve_id=vuln_id,
+vulnerability_id=vuln_id,
  vuln_references=vuln_reference)

vulnerabilities/importers/ruby.py

@@ -80,8 +80,7 @@ def collect_packages(self):
 
  def process_file(self, path) -> List[Advisory]:
  record = load_yaml(path)
- package_name = record.get(
- 'gem')
+ package_name = record.get('gem')
 
  if not package_name:
  return
@@ -131,7 +130,7 @@ def process_file(self, path) -> List[Advisory]:
  impacted_package_urls=impacted_purls,
  resolved_package_urls=resolved_purls,
  vuln_references=references,
-cve_id=cve_id
+vulnerability_id=cve_id
  )
 
  @staticmethod

vulnerabilities/importers/rust.py

@@ -135,7 +135,7 @@ def _load_advisory(self, path: str) -> Optional[Advisory]:
  summary=advisory.get("description", ""),
  impacted_package_urls=impacted_purls,
  resolved_package_urls=resolved_purls,
-cve_id=cve_id,
+vulnerability_id=cve_id,
  vuln_references=references,
  )