From 8b64a5f3d8478b1ede313cdd832253fb0b3dc364 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Tue, 21 Jul 2026 14:32:35 +0900 Subject: [PATCH 1/8] feat: replace Binary DB direct access with ldb_service API --- pyproject.toml | 1 - src/fosslight_binary/_binary_dao.py | 311 +++++++++++------------- src/fosslight_binary/_help.py | 7 +- src/fosslight_binary/binary_analysis.py | 4 +- src/fosslight_binary/cli.py | 18 +- 5 files changed, 166 insertions(+), 175 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 092e072..79c44e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ dependencies = [ "numpy", "pandas", "parmap", - "psycopg2-binary", "python-dateutil", "py-tlsh", "pytz", diff --git a/src/fosslight_binary/_binary_dao.py b/src/fosslight_binary/_binary_dao.py index 17932e1..9116f73 100755 --- a/src/fosslight_binary/_binary_dao.py +++ b/src/fosslight_binary/_binary_dao.py @@ -2,185 +2,168 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020 LG Electronics Inc. # SPDX-License-Identifier: Apache-2.0 +"""Binary DB lookup via ldb_service POST /binary/match.""" -import tlsh +import json import logging -import psycopg2 -import pandas as pd -import socket -from urllib.parse import urlparse +import os +import urllib.error +import urllib.request +from typing import Dict, List, Optional, Tuple + from fosslight_binary._binary import TLSH_CHECKSUM_NULL from fosslight_util.oss_item import OssItem import fosslight_util.constant as constant -columns = ['filename', 'pathname', 'checksum', 'tlshchecksum', 'ossname', - 'ossversion', 'license', 'platformname', - 'platformversion'] -conn = "" -cur = "" logger = logging.getLogger(constant.LOGGER_NAME) -DB_URL_DEFAULT = "postgresql://bin_analysis_script_user:script_123@bat.lge.com:5432/bat" - - -def get_oss_info_from_db(bin_info_list, dburl=""): - _cnt_auto_identified = 0 - conn_str, dbc = get_connection_string(dburl) - # DB URL에서 host 추출 - try: - db_host = dbc.hostname - except Exception as ex: - logger.warning(f"Failed to parse DB URL for host: {ex}") - db_host = None - is_internal = False - if db_host: - try: - # DNS lookup 시도 - socket.gethostbyname(db_host) - is_internal = True - except Exception: - is_internal = False - - if is_internal: - connect_to_lge_bin_db(conn_str) - if conn != "" and cur != "": - for item in bin_info_list: - bin_oss_items = [] - tlsh_value = item.tlsh - checksum_value = item.checksum - bin_file_name = item.binary_name_without_path - - df_result = get_oss_info_by_tlsh_and_filename( - bin_file_name, checksum_value, tlsh_value) - if df_result is not None and len(df_result) > 0: - _cnt_auto_identified += 1 - # Initialize the saved contents at .jar analyzing only once - if not item.found_in_jar_analysis and item.oss_items: - item.oss_items = [] - - for idx, row in df_result.iterrows(): - if not item.found_in_jar_analysis: - oss_from_db = OssItem(row['ossname'], row['ossversion'], row['license']) - - if bin_oss_items: - if not any(oss_item.name == oss_from_db.name - and oss_item.version == oss_from_db.version - and oss_item.license == oss_from_db.license - for oss_item in bin_oss_items): - bin_oss_items.append(oss_from_db) - else: - bin_oss_items.append(oss_from_db) - - if bin_oss_items: - item.set_oss_items(bin_oss_items) - item.comment = "Binary DB result" - item.found_in_binary = True +DEFAULT_KB_URL = "http://fosslight-kb.lge.com/" +_BINARY_MATCH_PATH = "/binary/match" +_HTTP_TIMEOUT_SEC = 120 +_CHUNK_SIZE = int(os.environ.get("BINARY_MATCH_CHUNK_SIZE", "3000")) + +MatchKey = Tuple[str, str] + + +def resolve_kb_config(kb_url: str = "", kb_token: str = "") -> Tuple[str, str]: + url = (kb_url or os.environ.get("KB_URL", DEFAULT_KB_URL)).strip() or DEFAULT_KB_URL + token = (kb_token or "").strip() or (os.environ.get("KB_TOKEN") or "").strip() + return f"{url.rstrip('/')}/", token + + +def _match_key(filename: str, checksum: str) -> MatchKey: + return filename, checksum or "" + + +def _build_deduped_payload( + bin_info_list, + tlsh_null: str, +) -> Tuple[List[dict], Dict[MatchKey, str]]: + """Deduplicate by filename+checksum; return API payload and key→api_id map.""" + key_to_id: Dict[MatchKey, str] = {} + items_payload: List[dict] = [] + + for item in bin_info_list: + filename = item.binary_name_without_path + checksum = item.checksum or "" + key = _match_key(filename, checksum) + if key in key_to_id: + continue + api_id = str(len(items_payload)) + key_to_id[key] = api_id + items_payload.append({ + "id": api_id, + "filename": filename, + "checksum": checksum, + "tlsh": item.tlsh or tlsh_null, + }) + + return items_payload, key_to_id + + +def _apply_match_result_to_item(item, result: dict) -> bool: + """Apply a /binary/match result to one binary item. Returns True if matched.""" + if not result or not result.get("matched"): + return False + oss_rows = result.get("oss_items") or [] + if not oss_rows: + return False + + if not item.found_in_jar_analysis and item.oss_items: + item.oss_items = [] + + bin_oss_items = [] + for row in oss_rows: + if item.found_in_jar_analysis: + break + oss_from_db = OssItem( + row.get("oss_name") or "", + row.get("oss_version") or "", + row.get("license") or "", + ) + if bin_oss_items: + if not any( + oss_item.name == oss_from_db.name + and oss_item.version == oss_from_db.version + and oss_item.license == oss_from_db.license + for oss_item in bin_oss_items + ): + bin_oss_items.append(oss_from_db) else: - logger.warning(f"Internal network detected, but DB connection to '{db_host}' failed. Skipping DB query.") - disconnect_lge_bin_db() - else: - logger.debug(f"Binary DB host '{db_host}' is not reachable. Skipping DB query.") - return bin_info_list, _cnt_auto_identified + bin_oss_items.append(oss_from_db) + + if bin_oss_items: + item.set_oss_items(bin_oss_items) + item.comment = "Binary DB result" + item.found_in_bin_db = True + return True + return False + + +def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): + """ + Call ldb_service /binary/match and attach OSS info to binary items. + Deduplicates by filename+checksum before the API call and maps results back. + Returns (bin_info_list, matched_count). + """ + _cnt_auto_identified = 0 + if not bin_info_list: + return bin_info_list, _cnt_auto_identified + base_url, token = resolve_kb_config(kb_url, kb_token) + items_payload, key_to_id = _build_deduped_payload(bin_info_list, TLSH_CHECKSUM_NULL) + if not items_payload: + return bin_info_list, _cnt_auto_identified -def get_connection_string(dburl): - # dburl format : 'postgresql://username:password@host:port/database_name' - connection_string = "" - user_dburl = True - dbc = "" - if dburl == "" or dburl is None: - user_dburl = False - dburl = DB_URL_DEFAULT + results_by_id = {} try: - if user_dburl: - logger.debug("DB URL:" + dburl) - dbc = urlparse(dburl) - connection_string = "dbname={dbname} user={user} host={host} password={password} port={port}" \ - .format(dbname=dbc.path.lstrip('/'), - user=dbc.username, - host=dbc.hostname, - password=dbc.password, - port=dbc.port) + for chunk_start in range(0, len(items_payload), _CHUNK_SIZE): + chunk = items_payload[chunk_start: chunk_start + _CHUNK_SIZE] + response = _post_binary_match(base_url, token, chunk) + if response is None: + return bin_info_list, _cnt_auto_identified + for result in response.get("results", []): + results_by_id[str(result.get("id"))] = result except Exception as ex: - if user_dburl: - logger.warning(f"(Minor) Failed to parsing db url : {ex}") - - return connection_string, dbc - - -def get_oss_info_by_tlsh_and_filename(file_name, checksum_value, tlsh_value): - sql_statement = "SELECT filename,pathname,checksum,tlshchecksum,ossname,ossversion,\ - license,platformname,platformversion FROM lgematching " - sql_statement_checksum = " WHERE filename=%s AND checksum=%s;" # Using parameterized query - sql_statement_filename = "SELECT DISTINCT ON (tlshchecksum) tlshchecksum FROM lgematching WHERE filename=%s;" # Using parameterized query - - final_result_item = "" - - df_result = get_list_by_using_query( - sql_statement + sql_statement_checksum, columns, (file_name, checksum_value)) - # Found a file with the same checksum. - if df_result is not None and len(df_result) > 0: - final_result_item = df_result - else: - # Match tlsh and fileName - df_result = get_list_by_using_query( - sql_statement_filename, ['tlshchecksum'], (file_name,)) - if df_result is None or len(df_result) <= 0: - final_result_item = "" - elif tlsh_value == TLSH_CHECKSUM_NULL: # Couldn't get the tlsh of a file. - final_result_item = "" - else: - matched_tlsh = "" - matched_tlsh_diff = -1 - for row in df_result.tlshchecksum: - try: - if row != TLSH_CHECKSUM_NULL: - tlsh_diff = tlsh.diff(row, tlsh_value) - if tlsh_diff <= 120: # MATCHED - if (matched_tlsh_diff < 0) or (tlsh_diff < matched_tlsh_diff): - matched_tlsh_diff = tlsh_diff - matched_tlsh = row - except Exception as ex: - logger.warning(f"* (Minor) Error_tlsh_comparison: {ex}") - if matched_tlsh != "": - final_result_item = get_list_by_using_query( - sql_statement + " WHERE filename=%s AND tlshchecksum=%s;", columns, (file_name, matched_tlsh)) - - return final_result_item - - -def get_list_by_using_query(sql_query, columns, params=None): - result_rows = "" # DataFrame - try: - if params: - cur.execute(sql_query, params) - else: - cur.execute(sql_query) - rows = cur.fetchall() - - if rows is not None and len(rows) > 0: - result_rows = pd.DataFrame(data=rows, columns=columns) - except Exception as ex: - logger.error(f"Database query error: {ex}") - result_rows = "" - return result_rows - + logger.warning(f"Binary match API failed: {ex}") + return bin_info_list, _cnt_auto_identified + + for item in bin_info_list: + key = _match_key(item.binary_name_without_path, item.checksum or "") + api_id = key_to_id.get(key) + if api_id is None: + continue + result = results_by_id.get(api_id) + if _apply_match_result_to_item(item, result): + _cnt_auto_identified += 1 -def disconnect_lge_bin_db(): - try: - cur.close() - conn.close() - except: - pass + return bin_info_list, _cnt_auto_identified -def connect_to_lge_bin_db(connection_string): - global conn, cur +def _post_binary_match(kb_url: str, kb_token: str, items: list) -> Optional[dict]: + data = json.dumps({"items": items}).encode("utf-8") + request = urllib.request.Request( + f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}", + data=data, + method="POST", + ) + request.add_header("Accept", "application/json") + request.add_header("Content-Type", "application/json") + if kb_token: + request.add_header("Authorization", f"Bearer {kb_token}") try: - conn = psycopg2.connect(connection_string) - cur = conn.cursor() - except Exception as ex: - logger.debug(f"(Minor) Can't connect to Binary DB. : {ex}") - conn = "" - cur = "" + with urllib.request.urlopen(request, timeout=_HTTP_TIMEOUT_SEC) as response: + body = response.read().decode() + return json.loads(body) if body else {} + except urllib.error.HTTPError as ex: + body = "" + try: + body = ex.read().decode() + except Exception: + pass + logger.warning(f"Binary match HTTP {ex.code}: {body or ex.reason}") + return None + except urllib.error.URLError as ex: + logger.debug(f"Binary match unreachable: {ex}") + return None diff --git a/src/fosslight_binary/_help.py b/src/fosslight_binary/_help.py index 8a9ef48..7045551 100644 --- a/src/fosslight_binary/_help.py +++ b/src/fosslight_binary/_help.py @@ -32,7 +32,8 @@ 🔍 Scanner-Specific Options ──────────────────────────────────────────────────────────────────── - -d DB Connection (format: 'postgresql://user:pass@host:port/db') + --kb_url KB API URL (priority: parameter > KB_URL env > default) + --kb_token KB bearer token (priority: parameter > KB_TOKEN env) --notice Print the open source license notice text --no_correction Skip OSS information correction with sbom-info.yaml --correct_fpath Path to custom sbom-info.yaml file @@ -48,8 +49,8 @@ # Generate output in specific format fosslight_binary -f excel -o results/ - # Connect to Binary DB for OSS information - fosslight_binary -d "postgresql://user:pass@localhost:5432/exampledb" + # Binary DB lookup via ldb_service + fosslight_binary --kb_url http://fosslight-kb.lge.com/ --kb_token """ diff --git a/src/fosslight_binary/binary_analysis.py b/src/fosslight_binary/binary_analysis.py index 7060e1e..d3fca34 100755 --- a/src/fosslight_binary/binary_analysis.py +++ b/src/fosslight_binary/binary_analysis.py @@ -169,7 +169,7 @@ def get_file_list(path_to_find, excluded_files): return file_cnt, bin_list, found_jar -def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=False, +def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="", simple_mode=False, correct_mode=True, correct_filepath="", path_to_exclude=[], all_exclude_mode=()): global start_time, finish_time, _root_path, _result_log @@ -254,7 +254,7 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F else: logger.warning("Could not find OSS information for some jar files.") - return_list, db_loaded_cnt = get_oss_info_from_db(return_list, dburl) + return_list, db_loaded_cnt = get_oss_info_from_db(return_list, kb_url, kb_token) return_list = sorted(return_list, key=lambda row: (row.bin_name_with_path)) scan_item.append_file_items(return_list, PKG_NAME) if correct_mode: diff --git a/src/fosslight_binary/cli.py b/src/fosslight_binary/cli.py index 56683d4..0ac2bba 100644 --- a/src/fosslight_binary/cli.py +++ b/src/fosslight_binary/cli.py @@ -36,7 +36,8 @@ def main(): path_to_exclude = [] output_dir = "" format = [] - db_url = "" + kb_url = "" + kb_token = "" simple_mode = False correct_mode = True @@ -46,9 +47,10 @@ def main(): parser.add_argument('-s', '--simple', action='store_true', required=False, help=argparse.SUPPRESS) parser.add_argument('-p', '--path', type=str, required=False) parser.add_argument('-o', '--output', type=str, required=False) - parser.add_argument('-d', '--dburl', type=str, default='', required=False) parser.add_argument('-f', '--formats', type=str, required=False, nargs="*") parser.add_argument('-e', '--exclude', nargs="*", required=False, default=[]) + parser.add_argument('--kb_url', type=str, required=False, default="") + parser.add_argument('--kb_token', type=str, required=False, default="") parser.add_argument('--notice', action='store_true', required=False) parser.add_argument('--no_correction', action='store_true', required=False) parser.add_argument('--correct_fpath', nargs=1, type=str, required=False) @@ -81,8 +83,11 @@ def main(): if args.output: # -o option output_dir = args.output - if args.dburl: # -d option - db_url = args.dburl + if args.kb_url: + kb_url = args.kb_url + + if args.kb_token: + kb_token = args.kb_token if args.formats: # -f option format = list(args.formats) @@ -113,7 +118,10 @@ def main(): timer.setDaemon(True) timer.start() - find_binaries(path_to_find_bin, output_dir, format, db_url, simple_mode, correct_mode, correct_filepath, path_to_exclude) + find_binaries( + path_to_find_bin, output_dir, format, kb_url, kb_token, + simple_mode, correct_mode, correct_filepath, path_to_exclude, + ) if __name__ == '__main__': From 8cdb5a6ae1a0f7417a7e8c95da867c77fdf380df Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Fri, 24 Jul 2026 14:46:40 +0900 Subject: [PATCH 2/8] feat: harden KB binary match with health and endpoint checks Skip API lookup when KB URL is invalid, /health fails, or /binary/match returns 404, and exclude flagged binaries from the match payload. --- src/fosslight_binary/_binary_dao.py | 97 ++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/src/fosslight_binary/_binary_dao.py b/src/fosslight_binary/_binary_dao.py index 9116f73..4e35249 100755 --- a/src/fosslight_binary/_binary_dao.py +++ b/src/fosslight_binary/_binary_dao.py @@ -7,9 +7,11 @@ import json import logging import os +import time import urllib.error import urllib.request from typing import Dict, List, Optional, Tuple +from urllib.parse import urlparse from fosslight_binary._binary import TLSH_CHECKSUM_NULL from fosslight_util.oss_item import OssItem @@ -20,6 +22,7 @@ DEFAULT_KB_URL = "http://fosslight-kb.lge.com/" _BINARY_MATCH_PATH = "/binary/match" _HTTP_TIMEOUT_SEC = 120 +_HEALTH_TIMEOUT_SEC = 10 _CHUNK_SIZE = int(os.environ.get("BINARY_MATCH_CHUNK_SIZE", "3000")) MatchKey = Tuple[str, str] @@ -31,6 +34,82 @@ def resolve_kb_config(kb_url: str = "", kb_token: str = "") -> Tuple[str, str]: return f"{url.rstrip('/')}/", token +def _is_valid_kb_url_format(kb_url: str) -> bool: + parsed = urlparse(kb_url) + return parsed.scheme in ("http", "https") and bool(parsed.netloc) + + +def check_kb_server_reachable(kb_url: str, kb_token: str = "") -> bool: + """Return True if KB URL format is valid and /health is reachable.""" + if not _is_valid_kb_url_format(kb_url): + logger.warning(f"Invalid KB URL format: {kb_url}") + return False + + for attempt in range(3): + try: + request = urllib.request.Request(f"{kb_url}health", method="GET") + if kb_token: + request.add_header("Authorization", f"Bearer {kb_token}") + with urllib.request.urlopen(request, timeout=_HEALTH_TIMEOUT_SEC) as response: + logger.debug(f"KB server is reachable. Response status: {response.status}") + return True + except urllib.error.HTTPError: + logger.debug("KB server responded (HTTP error), considered reachable") + return True + except urllib.error.URLError as ex: + logger.debug(f"KB server is unreachable: {ex}") + if attempt < 2: + time.sleep(1) + else: + return False + except Exception as ex: + logger.debug(f"Unexpected error checking KB server: {ex}") + if attempt < 2: + time.sleep(1) + else: + return False + return False + + +def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> bool: + """ + Return True if POST /binary/match exists. + /health can succeed on a host that does not expose /binary/match (404). + Other HTTP statuses (401/400/503/…) mean the route is present. + """ + endpoint = f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}" + data = json.dumps({"items": []}).encode("utf-8") + request = urllib.request.Request(endpoint, data=data, method="POST") + request.add_header("Accept", "application/json") + request.add_header("Content-Type", "application/json") + if kb_token: + request.add_header("Authorization", f"Bearer {kb_token}") + + try: + with urllib.request.urlopen(request, timeout=_HEALTH_TIMEOUT_SEC) as response: + response.read() + return True + except urllib.error.HTTPError as ex: + if ex.code == 404: + logger.warning( + f"KB binary match endpoint not found (HTTP 404): {endpoint}. " + "Skipping binary match API." + ) + return False + logger.debug( + f"KB binary match endpoint responded with HTTP {ex.code}; treated as available" + ) + return True + except urllib.error.URLError as ex: + logger.warning(f"KB binary match endpoint unreachable: {ex}. Skipping binary match API.") + return False + except Exception as ex: + logger.warning( + f"Failed to check KB binary match endpoint: {ex}. Skipping binary match API." + ) + return False + + def _match_key(filename: str, checksum: str) -> MatchKey: return filename, checksum or "" @@ -44,6 +123,8 @@ def _build_deduped_payload( items_payload: List[dict] = [] for item in bin_info_list: + if item.exclude: + continue filename = item.binary_name_without_path checksum = item.checksum or "" key = _match_key(filename, checksum) @@ -111,6 +192,12 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): return bin_info_list, _cnt_auto_identified base_url, token = resolve_kb_config(kb_url, kb_token) + if not check_kb_server_reachable(base_url, token): + logger.warning(f"KB({base_url}) Unreachable. Skipping binary match API.") + return bin_info_list, _cnt_auto_identified + if not check_binary_match_endpoint(base_url, token): + return bin_info_list, _cnt_auto_identified + items_payload, key_to_id = _build_deduped_payload(bin_info_list, TLSH_CHECKSUM_NULL) if not items_payload: return bin_info_list, _cnt_auto_identified @@ -129,6 +216,8 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): return bin_info_list, _cnt_auto_identified for item in bin_info_list: + if item.exclude: + continue key = _match_key(item.binary_name_without_path, item.checksum or "") api_id = key_to_id.get(key) if api_id is None: @@ -162,7 +251,13 @@ def _post_binary_match(kb_url: str, kb_token: str, items: list) -> Optional[dict body = ex.read().decode() except Exception: pass - logger.warning(f"Binary match HTTP {ex.code}: {body or ex.reason}") + if ex.code == 404: + logger.warning( + f"Binary match endpoint not found (HTTP 404): " + f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}" + ) + else: + logger.warning(f"Binary match HTTP {ex.code}: {body or ex.reason}") return None except urllib.error.URLError as ex: logger.debug(f"Binary match unreachable: {ex}") From 4bcc9506f70b7b595fc33c393534291d969b4785 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Fri, 24 Jul 2026 15:05:36 +0900 Subject: [PATCH 3/8] fix: validate BINARY_MATCH_CHUNK_SIZE env value safely Fall back to 3000 with a warning when the env var is non-numeric or <= 0. --- src/fosslight_binary/_binary_dao.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/fosslight_binary/_binary_dao.py b/src/fosslight_binary/_binary_dao.py index 4e35249..dc181b5 100755 --- a/src/fosslight_binary/_binary_dao.py +++ b/src/fosslight_binary/_binary_dao.py @@ -23,7 +23,25 @@ _BINARY_MATCH_PATH = "/binary/match" _HTTP_TIMEOUT_SEC = 120 _HEALTH_TIMEOUT_SEC = 10 -_CHUNK_SIZE = int(os.environ.get("BINARY_MATCH_CHUNK_SIZE", "3000")) +_DEFAULT_CHUNK_SIZE = 3000 + + +def _get_chunk_size() -> int: + raw = os.environ.get("BINARY_MATCH_CHUNK_SIZE", str(_DEFAULT_CHUNK_SIZE)) + try: + chunk_size = int(raw) + except ValueError: + logger.warning(f"Invalid BINARY_MATCH_CHUNK_SIZE={raw!r}; using {_DEFAULT_CHUNK_SIZE}") + return _DEFAULT_CHUNK_SIZE + if chunk_size <= 0: + logger.warning( + f"BINARY_MATCH_CHUNK_SIZE must be > 0 (got {chunk_size}); using {_DEFAULT_CHUNK_SIZE}" + ) + return _DEFAULT_CHUNK_SIZE + return chunk_size + + +_CHUNK_SIZE = _get_chunk_size() MatchKey = Tuple[str, str] From c126f80589fbcd64484860a85d4b303ee3b3162d Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Mon, 27 Jul 2026 08:01:14 +0900 Subject: [PATCH 4/8] fix: keep partial binary match results and probe only /binary/match Drop the /health pre-check, continue remaining chunks after a failed chunk, and still apply results collected so far. --- src/fosslight_binary/_binary_dao.py | 56 ++++++++--------------------- 1 file changed, 14 insertions(+), 42 deletions(-) diff --git a/src/fosslight_binary/_binary_dao.py b/src/fosslight_binary/_binary_dao.py index dc181b5..7f03ee3 100755 --- a/src/fosslight_binary/_binary_dao.py +++ b/src/fosslight_binary/_binary_dao.py @@ -7,7 +7,6 @@ import json import logging import os -import time import urllib.error import urllib.request from typing import Dict, List, Optional, Tuple @@ -22,7 +21,7 @@ DEFAULT_KB_URL = "http://fosslight-kb.lge.com/" _BINARY_MATCH_PATH = "/binary/match" _HTTP_TIMEOUT_SEC = 120 -_HEALTH_TIMEOUT_SEC = 10 +_PROBE_TIMEOUT_SEC = 10 _DEFAULT_CHUNK_SIZE = 3000 @@ -57,44 +56,16 @@ def _is_valid_kb_url_format(kb_url: str) -> bool: return parsed.scheme in ("http", "https") and bool(parsed.netloc) -def check_kb_server_reachable(kb_url: str, kb_token: str = "") -> bool: - """Return True if KB URL format is valid and /health is reachable.""" +def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> bool: + """ + Return True if URL is valid and POST /binary/match is available. + Probes with empty {"items": []}. HTTP 404 means the route is missing; + other HTTP statuses (401/400/503/…) mean the route is present. + """ if not _is_valid_kb_url_format(kb_url): logger.warning(f"Invalid KB URL format: {kb_url}") return False - for attempt in range(3): - try: - request = urllib.request.Request(f"{kb_url}health", method="GET") - if kb_token: - request.add_header("Authorization", f"Bearer {kb_token}") - with urllib.request.urlopen(request, timeout=_HEALTH_TIMEOUT_SEC) as response: - logger.debug(f"KB server is reachable. Response status: {response.status}") - return True - except urllib.error.HTTPError: - logger.debug("KB server responded (HTTP error), considered reachable") - return True - except urllib.error.URLError as ex: - logger.debug(f"KB server is unreachable: {ex}") - if attempt < 2: - time.sleep(1) - else: - return False - except Exception as ex: - logger.debug(f"Unexpected error checking KB server: {ex}") - if attempt < 2: - time.sleep(1) - else: - return False - return False - - -def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> bool: - """ - Return True if POST /binary/match exists. - /health can succeed on a host that does not expose /binary/match (404). - Other HTTP statuses (401/400/503/…) mean the route is present. - """ endpoint = f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}" data = json.dumps({"items": []}).encode("utf-8") request = urllib.request.Request(endpoint, data=data, method="POST") @@ -104,7 +75,7 @@ def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> bool: request.add_header("Authorization", f"Bearer {kb_token}") try: - with urllib.request.urlopen(request, timeout=_HEALTH_TIMEOUT_SEC) as response: + with urllib.request.urlopen(request, timeout=_PROBE_TIMEOUT_SEC) as response: response.read() return True except urllib.error.HTTPError as ex: @@ -210,9 +181,6 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): return bin_info_list, _cnt_auto_identified base_url, token = resolve_kb_config(kb_url, kb_token) - if not check_kb_server_reachable(base_url, token): - logger.warning(f"KB({base_url}) Unreachable. Skipping binary match API.") - return bin_info_list, _cnt_auto_identified if not check_binary_match_endpoint(base_url, token): return bin_info_list, _cnt_auto_identified @@ -226,12 +194,16 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): chunk = items_payload[chunk_start: chunk_start + _CHUNK_SIZE] response = _post_binary_match(base_url, token, chunk) if response is None: - return bin_info_list, _cnt_auto_identified + logger.warning( + f"Binary match chunk failed " + f"({chunk_start}:{chunk_start + len(chunk)}); " + "keeping results so far and continuing with next chunks." + ) + continue for result in response.get("results", []): results_by_id[str(result.get("id"))] = result except Exception as ex: logger.warning(f"Binary match API failed: {ex}") - return bin_info_list, _cnt_auto_identified for item in bin_info_list: if item.exclude: From bbf84dd7b9c96919837bd0a11a7cf8e4a21eeb38 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Fri, 31 Jul 2026 10:36:19 +0900 Subject: [PATCH 5/8] fix(binary): do not dedupe KB match items with unknown checksum Keep separate /binary/match entries when checksum is empty or "0" so same-basename binaries are not given identical OSS attribution. --- src/fosslight_binary/_binary_dao.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/fosslight_binary/_binary_dao.py b/src/fosslight_binary/_binary_dao.py index 7f03ee3..9730628 100755 --- a/src/fosslight_binary/_binary_dao.py +++ b/src/fosslight_binary/_binary_dao.py @@ -99,25 +99,37 @@ def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> bool: return False -def _match_key(filename: str, checksum: str) -> MatchKey: - return filename, checksum or "" +def _is_unknown_checksum(checksum: str) -> bool: + """True when checksum was not computed (empty or TLSH_CHECKSUM_NULL).""" + return (not checksum) or checksum == TLSH_CHECKSUM_NULL + + +def _match_key(filename: str, checksum: str, index: int) -> MatchKey: + """Dedupe key. Unknown checksums stay unique per list index (no filename-only merge).""" + if _is_unknown_checksum(checksum): + return filename, f"__unknown_{index}" + return filename, checksum def _build_deduped_payload( bin_info_list, tlsh_null: str, ) -> Tuple[List[dict], Dict[MatchKey, str]]: - """Deduplicate by filename+checksum; return API payload and key→api_id map.""" + """Deduplicate by filename+checksum; return API payload and key→api_id map. + + Items with empty/\"0\" checksum are not deduped — each keeps its own API entry + (and its own tlsh) so distinct files that share a basename are not conflated. + """ key_to_id: Dict[MatchKey, str] = {} items_payload: List[dict] = [] - for item in bin_info_list: + for index, item in enumerate(bin_info_list): if item.exclude: continue filename = item.binary_name_without_path checksum = item.checksum or "" - key = _match_key(filename, checksum) - if key in key_to_id: + key = _match_key(filename, checksum, index) + if not _is_unknown_checksum(checksum) and key in key_to_id: continue api_id = str(len(items_payload)) key_to_id[key] = api_id @@ -205,10 +217,10 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): except Exception as ex: logger.warning(f"Binary match API failed: {ex}") - for item in bin_info_list: + for index, item in enumerate(bin_info_list): if item.exclude: continue - key = _match_key(item.binary_name_without_path, item.checksum or "") + key = _match_key(item.binary_name_without_path, item.checksum or "", index) api_id = key_to_id.get(key) if api_id is None: continue From 2791f77e8407fdc9cf951cde94132040f2143354 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Fri, 31 Jul 2026 10:42:43 +0900 Subject: [PATCH 6/8] fix(binary): skip KB match when checksum and tlsh are both unknown Omit /binary/match payload entries that have empty/"0" checksum and tlsh, since there is nothing useful to match against. --- src/fosslight_binary/_binary_dao.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/fosslight_binary/_binary_dao.py b/src/fosslight_binary/_binary_dao.py index 9730628..f64bfad 100755 --- a/src/fosslight_binary/_binary_dao.py +++ b/src/fosslight_binary/_binary_dao.py @@ -104,6 +104,11 @@ def _is_unknown_checksum(checksum: str) -> bool: return (not checksum) or checksum == TLSH_CHECKSUM_NULL +def _is_unknown_tlsh(tlsh: str) -> bool: + """True when tlsh was not computed (empty or TLSH_CHECKSUM_NULL).""" + return (not tlsh) or tlsh == TLSH_CHECKSUM_NULL + + def _match_key(filename: str, checksum: str, index: int) -> MatchKey: """Dedupe key. Unknown checksums stay unique per list index (no filename-only merge).""" if _is_unknown_checksum(checksum): @@ -119,6 +124,7 @@ def _build_deduped_payload( Items with empty/\"0\" checksum are not deduped — each keeps its own API entry (and its own tlsh) so distinct files that share a basename are not conflated. + Items with both checksum and tlsh unknown are omitted (nothing to match). """ key_to_id: Dict[MatchKey, str] = {} items_payload: List[dict] = [] @@ -128,6 +134,9 @@ def _build_deduped_payload( continue filename = item.binary_name_without_path checksum = item.checksum or "" + tlsh = item.tlsh or tlsh_null + if _is_unknown_checksum(checksum) and _is_unknown_tlsh(tlsh): + continue key = _match_key(filename, checksum, index) if not _is_unknown_checksum(checksum) and key in key_to_id: continue @@ -137,7 +146,7 @@ def _build_deduped_payload( "id": api_id, "filename": filename, "checksum": checksum, - "tlsh": item.tlsh or tlsh_null, + "tlsh": tlsh, }) return items_payload, key_to_id From 9911791f413b1ada18958667423f95db2426dec7 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Mon, 3 Aug 2026 06:23:58 +0900 Subject: [PATCH 7/8] feat(binary): add KB Unreachable to Scanner Info comment When /binary/match probe cannot reach the host, record KB({kb_url}) Unreachable on the cover sheet like source scanner. --- src/fosslight_binary/_binary_dao.py | 34 +++++++++++++------------ src/fosslight_binary/binary_analysis.py | 4 ++- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/fosslight_binary/_binary_dao.py b/src/fosslight_binary/_binary_dao.py index f64bfad..98da968 100755 --- a/src/fosslight_binary/_binary_dao.py +++ b/src/fosslight_binary/_binary_dao.py @@ -56,15 +56,16 @@ def _is_valid_kb_url_format(kb_url: str) -> bool: return parsed.scheme in ("http", "https") and bool(parsed.netloc) -def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> bool: +def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> Tuple[bool, str]: """ - Return True if URL is valid and POST /binary/match is available. - Probes with empty {"items": []}. HTTP 404 means the route is missing; - other HTTP statuses (401/400/503/…) mean the route is present. + Return (available, cover_comment). + + cover_comment is ``KB({kb_url}) Unreachable`` when the host cannot be reached + (URLError / connection failure). Empty for other skip cases (invalid URL, HTTP 404). """ if not _is_valid_kb_url_format(kb_url): logger.warning(f"Invalid KB URL format: {kb_url}") - return False + return False, "" endpoint = f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}" data = json.dumps({"items": []}).encode("utf-8") @@ -77,26 +78,26 @@ def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> bool: try: with urllib.request.urlopen(request, timeout=_PROBE_TIMEOUT_SEC) as response: response.read() - return True + return True, "" except urllib.error.HTTPError as ex: if ex.code == 404: logger.warning( f"KB binary match endpoint not found (HTTP 404): {endpoint}. " "Skipping binary match API." ) - return False + return False, "" logger.debug( f"KB binary match endpoint responded with HTTP {ex.code}; treated as available" ) - return True + return True, "" except urllib.error.URLError as ex: logger.warning(f"KB binary match endpoint unreachable: {ex}. Skipping binary match API.") - return False + return False, f"KB({kb_url}) Unreachable" except Exception as ex: logger.warning( f"Failed to check KB binary match endpoint: {ex}. Skipping binary match API." ) - return False + return False, f"KB({kb_url}) Unreachable" def _is_unknown_checksum(checksum: str) -> bool: @@ -195,19 +196,20 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): """ Call ldb_service /binary/match and attach OSS info to binary items. Deduplicates by filename+checksum before the API call and maps results back. - Returns (bin_info_list, matched_count). + Returns (bin_info_list, matched_count, kb_cover_comment). """ _cnt_auto_identified = 0 if not bin_info_list: - return bin_info_list, _cnt_auto_identified + return bin_info_list, _cnt_auto_identified, "" base_url, token = resolve_kb_config(kb_url, kb_token) - if not check_binary_match_endpoint(base_url, token): - return bin_info_list, _cnt_auto_identified + available, kb_cover_comment = check_binary_match_endpoint(base_url, token) + if not available: + return bin_info_list, _cnt_auto_identified, kb_cover_comment items_payload, key_to_id = _build_deduped_payload(bin_info_list, TLSH_CHECKSUM_NULL) if not items_payload: - return bin_info_list, _cnt_auto_identified + return bin_info_list, _cnt_auto_identified, "" results_by_id = {} try: @@ -237,7 +239,7 @@ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""): if _apply_match_result_to_item(item, result): _cnt_auto_identified += 1 - return bin_info_list, _cnt_auto_identified + return bin_info_list, _cnt_auto_identified, "" def _post_binary_match(kb_url: str, kb_token: str, items: list) -> Optional[dict]: diff --git a/src/fosslight_binary/binary_analysis.py b/src/fosslight_binary/binary_analysis.py index d3fca34..b883ad1 100755 --- a/src/fosslight_binary/binary_analysis.py +++ b/src/fosslight_binary/binary_analysis.py @@ -254,7 +254,7 @@ def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="", else: logger.warning("Could not find OSS information for some jar files.") - return_list, db_loaded_cnt = get_oss_info_from_db(return_list, kb_url, kb_token) + return_list, db_loaded_cnt, kb_cover_msg = get_oss_info_from_db(return_list, kb_url, kb_token) return_list = sorted(return_list, key=lambda row: (row.bin_name_with_path)) scan_item.append_file_items(return_list, PKG_NAME) if correct_mode: @@ -267,6 +267,8 @@ def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="", finish_time = current_timestamp_utc() scan_item.set_cover_comment(f"Detected binaries: {len(return_list)} (Scanned Files : {cnt_file_except_skipped})") + if kb_cover_msg: + scan_item.set_cover_comment(kb_cover_msg) scan_item.set_cover_finish_time(finish_time) for combined_path_and_file, output_extension, output_format in zip(result_reports, output_extensions, formats): From e75ca82375d0164c17b8c73303938e02662f2deb Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Mon, 3 Aug 2026 06:26:18 +0900 Subject: [PATCH 8/8] feat(binary): add KB Skipped cover comment on HTTP 404 When /binary/match probe returns 404, record KB({kb_url}) Skipped on the Scanner Info cover comment instead of leaving it empty. --- src/fosslight_binary/_binary_dao.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/fosslight_binary/_binary_dao.py b/src/fosslight_binary/_binary_dao.py index 98da968..4b34a4c 100755 --- a/src/fosslight_binary/_binary_dao.py +++ b/src/fosslight_binary/_binary_dao.py @@ -60,8 +60,10 @@ def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> Tuple[bool, """ Return (available, cover_comment). - cover_comment is ``KB({kb_url}) Unreachable`` when the host cannot be reached - (URLError / connection failure). Empty for other skip cases (invalid URL, HTTP 404). + cover_comment: + - ``KB({kb_url}) Unreachable`` when the host cannot be reached + - ``KB({kb_url}) Skipped`` when /binary/match is missing (HTTP 404) + - empty for other skip cases (invalid URL) """ if not _is_valid_kb_url_format(kb_url): logger.warning(f"Invalid KB URL format: {kb_url}") @@ -85,7 +87,7 @@ def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> Tuple[bool, f"KB binary match endpoint not found (HTTP 404): {endpoint}. " "Skipping binary match API." ) - return False, "" + return False, f"KB({kb_url}) Skipped" logger.debug( f"KB binary match endpoint responded with HTTP {ex.code}; treated as available" )