diff --git a/.gitignore b/.gitignore index c8a45f86a..4e7c6220c 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,5 @@ tmp/ cres/* ### Local project management tooling project management scripts/ + +.harvester_cache/ diff --git a/application/database/db.py b/application/database/db.py index 4ca0dc040..1969e9e9b 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -8,6 +8,7 @@ import time import yaml +from datetime import datetime, timezone from pprint import pprint from collections import Counter, defaultdict @@ -257,6 +258,108 @@ class StagedChangeSet(BaseModel): # type: ignore created_at = sqla.Column(sqla.DateTime, nullable=False) +class ArtifactIngestEvent(BaseModel): # type: ignore + """Tracks one harvested artifact persisted per import run.""" + + __tablename__ = "artifact_ingest_event" + id = sqla.Column(sqla.String, primary_key=True, default=generate_uuid) + run_id = sqla.Column( + sqla.String, + sqla.ForeignKey("import_run.id", onupdate="CASCADE", ondelete="CASCADE"), + nullable=False, + ) + artifact_id = sqla.Column(sqla.String, nullable=False) + harvest_mode = sqla.Column(sqla.String, nullable=False) + event_type = sqla.Column(sqla.String, nullable=False) + source_json = sqla.Column(sqla.Text, nullable=False) + locator_json = sqla.Column(sqla.Text, nullable=False) + artifact_json = sqla.Column(sqla.Text, nullable=False) + harvest_json = sqla.Column(sqla.Text, nullable=False) + observed_at = sqla.Column(sqla.DateTime, nullable=False) + created_at = sqla.Column(sqla.DateTime, nullable=False) + + __table_args__ = ( + sqla.UniqueConstraint( + run_id, + artifact_id, + name="uq_artifact_ingest_event_run_artifact", + ), + ) + + +class IngestChunk(BaseModel): # type: ignore + """Tracks every chunk belonging to an artifact ingest event.""" + + __tablename__ = "ingest_chunk" + id = sqla.Column(sqla.String, primary_key=True, default=generate_uuid) + artifact_event_id = sqla.Column( + sqla.String, + sqla.ForeignKey( + "artifact_ingest_event.id", + onupdate="CASCADE", + ondelete="CASCADE", + ), + nullable=False, + ) + chunk_id = sqla.Column(sqla.String, nullable=False) + text = sqla.Column(sqla.Text, nullable=False) + char_count = sqla.Column(sqla.Integer, nullable=False) + span_json = sqla.Column(sqla.Text, nullable=False) + delta_json = sqla.Column(sqla.Text, nullable=True) + created_at = sqla.Column(sqla.DateTime, nullable=False) + + __table_args__ = ( + sqla.UniqueConstraint( + artifact_event_id, + chunk_id, + name="uq_ingest_chunk_artifact_chunk", + ), + ) + + +class HarvesterCheckpoint(BaseModel): # type: ignore + __tablename__ = "harvester_checkpoint" + repository_id = sqla.Column(sqla.String, primary_key=True) + provider = sqla.Column(sqla.String, nullable=False) + owner = sqla.Column(sqla.String, nullable=False) + repository = sqla.Column(sqla.String, nullable=False) + branch = sqla.Column(sqla.String, nullable=False) + last_processed_commit = sqla.Column(sqla.String, nullable=True) + created_at = sqla.Column( + sqla.DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + updated_at = sqla.Column( + sqla.DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + __table_args__ = ( + sqla.UniqueConstraint( + "provider", + "owner", + "repository", + "branch", + name="uq_harvester_checkpoint_canonical_source", + ), + ) + + +def _serialize_json_value(value: Any) -> str: + return flask_json.dumps(value) + + +def _normalize_utc_datetime(value: Any) -> Any: + from datetime import datetime, timezone + + if isinstance(value, datetime): + if value.tzinfo is None: + return value + return value.astimezone(timezone.utc) + return value + + def create_import_run(source: str, version: Optional[str] = None) -> ImportRun: """Create and persist an import run record. Returns the new ImportRun.""" from datetime import datetime, timezone @@ -304,6 +407,68 @@ def get_previous_import_run(source: str, current_run_id: str) -> Optional[Import ) +def create_artifact_ingest_event( + *, + run_id: str, + artifact_id: str, + harvest_mode: str, + event_type: str, + source_json: Any, + locator_json: Any, + artifact_json: Any, + harvest_json: Any, + observed_at: Any, +) -> ArtifactIngestEvent: + from datetime import datetime, timezone + + observed_at = _normalize_utc_datetime(observed_at) + + event = ArtifactIngestEvent( + id=generate_uuid(), + run_id=run_id, + artifact_id=artifact_id, + harvest_mode=harvest_mode, + event_type=event_type, + source_json=_serialize_json_value(source_json), + locator_json=_serialize_json_value(locator_json), + artifact_json=_serialize_json_value(artifact_json), + harvest_json=_serialize_json_value(harvest_json), + observed_at=observed_at, + created_at=_normalize_utc_datetime(datetime.now(timezone.utc)), + ) + sqla.session.add(event) + sqla.session.commit() + return event + + +def create_ingest_chunk( + *, + artifact_event_id: str, + chunk_id: str, + text: str, + char_count: int, + span_json: Any, + delta_json: Optional[Any] = None, +) -> IngestChunk: + from datetime import datetime, timezone + + chunk = IngestChunk( + id=generate_uuid(), + artifact_event_id=artifact_event_id, + chunk_id=chunk_id, + text=text, + char_count=char_count, + span_json=_serialize_json_value(span_json), + delta_json=( + _serialize_json_value(delta_json) if delta_json is not None else None + ), + created_at=_normalize_utc_datetime(datetime.now(timezone.utc)), + ) + sqla.session.add(chunk) + sqla.session.commit() + return chunk + + def persist_standard_snapshot( *, run_id: str, diff --git a/application/tests/harvester_test/change_detector_test.py b/application/tests/harvester_test/change_detector_test.py new file mode 100644 index 000000000..5cb33456a --- /dev/null +++ b/application/tests/harvester_test/change_detector_test.py @@ -0,0 +1,168 @@ +import unittest +from unittest.mock import MagicMock +from unittest.mock import call +from unittest.mock import patch + +from application.utils.harvester.change_detector import ( + ChangeDetector, +) + + +class ChangeDetectorTests(unittest.TestCase): + @patch("application.utils.harvester.change_detector.subprocess.run") + def test_get_modified_files_since(self, mock_run): + client = MagicMock() + client.get_local_path.return_value = "repo-under-test" + + mock_run.side_effect = [ + MagicMock(stdout="resolved_base\n"), + MagicMock(stdout="resolved_target\n"), + MagicMock(stdout="a.md\nb.md\na.md\n"), + ] + + detector = ChangeDetector(client) + + files = detector.get_modified_files_since( + "base", + "target", + ) + + self.assertEqual( + files, + [ + "a.md", + "b.md", + ], + ) + + mock_run.assert_has_calls( + [ + call( + [ + "git", + "-C", + "repo-under-test", + "rev-parse", + "--verify", + "--end-of-options", + "base^{commit}", + ], + capture_output=True, + text=True, + check=True, + timeout=60, + ), + call( + [ + "git", + "-C", + "repo-under-test", + "rev-parse", + "--verify", + "--end-of-options", + "target^{commit}", + ], + capture_output=True, + text=True, + check=True, + timeout=60, + ), + call( + [ + "git", + "-C", + "repo-under-test", + "diff", + "--name-only", + "resolved_base", + "resolved_target", + ], + capture_output=True, + text=True, + check=True, + timeout=60, + ), + ] + ) + + @patch("application.utils.harvester.change_detector.subprocess.run") + def test_get_commits_since(self, mock_run): + client = MagicMock() + client.get_local_path.return_value = "repo-under-test" + + mock_run.side_effect = [ + MagicMock(stdout="resolved_base\n"), + MagicMock(stdout="resolved_target\n"), + MagicMock(stdout="111\n222\n333\n"), + ] + + detector = ChangeDetector(client) + + commits = detector.get_commits_since( + "base", + "target", + ) + + self.assertEqual( + commits, + [ + "111", + "222", + "333", + ], + ) + + self.assertEqual( + mock_run.call_args_list, + [ + call( + [ + "git", + "-C", + "repo-under-test", + "rev-parse", + "--verify", + "--end-of-options", + "base^{commit}", + ], + capture_output=True, + text=True, + check=True, + timeout=60, + ), + call( + [ + "git", + "-C", + "repo-under-test", + "rev-parse", + "--verify", + "--end-of-options", + "target^{commit}", + ], + capture_output=True, + text=True, + check=True, + timeout=60, + ), + call( + [ + "git", + "-C", + "repo-under-test", + "log", + "--reverse", + "--format=%H", + "resolved_base..resolved_target", + ], + capture_output=True, + text=True, + check=True, + timeout=60, + ), + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/checkpoint_store_test.py b/application/tests/harvester_test/checkpoint_store_test.py new file mode 100644 index 000000000..5d5f1e947 --- /dev/null +++ b/application/tests/harvester_test/checkpoint_store_test.py @@ -0,0 +1,193 @@ +import unittest +from datetime import datetime, timezone + +from application import create_app, sqla +from application.utils.harvester.checkpoint_store import CheckpointStore +from application.utils.harvester.models import RepositoryCheckpoint + + +class CheckpointStoreTests(unittest.TestCase): + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def test_save_and_load_checkpoint(self): + store = CheckpointStore() + checkpoint = RepositoryCheckpoint( + repository_id="owasp-asvs", + last_processed_commit="abc123", + updated_at=datetime.now(timezone.utc), + provider="github", + owner="owasp", + repository="asvs", + branch="main", + ) + + store.save(checkpoint) + loaded = store.load("owasp-asvs") + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual(loaded.last_processed_commit, "abc123") + self.assertEqual(loaded.provider, "github") + + def test_update_upsert_and_two_repositories_remain_isolated(self): + store = CheckpointStore() + repo_a = RepositoryCheckpoint( + repository_id="repo-a", + last_processed_commit="commit-1", + updated_at=datetime.now(timezone.utc), + provider="github", + owner="sample", + repository="repo-a", + branch="main", + ) + repo_b = RepositoryCheckpoint( + repository_id="repo-b", + last_processed_commit="commit-b", + updated_at=datetime.now(timezone.utc), + provider="github", + owner="sample", + repository="repo-b", + branch="main", + ) + store.save(repo_a) + store.save(repo_b) + + updated_a = RepositoryCheckpoint( + repository_id="repo-a", + last_processed_commit="commit-2", + updated_at=datetime.now(timezone.utc), + provider="github", + owner="sample", + repository="repo-a", + branch="main", + ) + store.save(updated_a) + + loaded_a = store.load("repo-a") + loaded_b = store.load("repo-b") + + self.assertIsNotNone(loaded_a) + self.assertIsNotNone(loaded_b) + assert loaded_a is not None + assert loaded_b is not None + self.assertEqual(loaded_a.last_processed_commit, "commit-2") + self.assertEqual(loaded_b.last_processed_commit, "commit-b") + + def test_duplicate_canonical_source_identity_rejected(self): + store = CheckpointStore() + first = RepositoryCheckpoint( + repository_id="repo-a", + last_processed_commit="commit-1", + updated_at=datetime.now(timezone.utc), + provider="github", + owner="sample", + repository="shared", + branch="main", + ) + second = RepositoryCheckpoint( + repository_id="repo-b", + last_processed_commit="commit-2", + updated_at=datetime.now(timezone.utc), + provider="github", + owner="sample", + repository="shared", + branch="main", + ) + + store.save(first) + + with self.assertRaises(ValueError): + store.save(second) + + def test_immutable_repository_identity(self): + store = CheckpointStore() + first = RepositoryCheckpoint( + repository_id="repo-a", + last_processed_commit="commit-1", + updated_at=datetime.now(timezone.utc), + provider="github", + owner="sample", + repository="repo-a", + branch="main", + ) + store.save(first) + + conflicting = RepositoryCheckpoint( + repository_id="repo-a", + last_processed_commit="commit-2", + updated_at=datetime.now(timezone.utc), + provider="github", + owner="sample", + repository="repo-a", + branch="develop", + ) + + with self.assertRaises(ValueError): + store.save(conflicting) + + def test_null_initial_checkpoint(self): + store = CheckpointStore() + checkpoint = RepositoryCheckpoint( + repository_id="repo-a", + last_processed_commit=None, + updated_at=datetime.now(timezone.utc), + provider="github", + owner="sample", + repository="repo-a", + branch="main", + ) + + store.save(checkpoint) + loaded = store.load("repo-a") + + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertIsNone(loaded.last_processed_commit) + + def test_transaction_rollback_leaves_previous_checkpoint_intact(self): + store = CheckpointStore() + original = RepositoryCheckpoint( + repository_id="repo-a", + last_processed_commit="commit-1", + updated_at=datetime.now(timezone.utc), + provider="github", + owner="sample", + repository="repo-a", + branch="main", + ) + store.save(original) + + conflicting = RepositoryCheckpoint( + repository_id="repo-a", + last_processed_commit="commit-2", + updated_at=datetime.now(timezone.utc), + provider="github", + owner="sample", + repository="repo-a", + branch="develop", + ) + + with self.assertRaises(ValueError): + store.save(conflicting) + + loaded = store.load("repo-a") + self.assertIsNotNone(loaded) + assert loaded is not None + self.assertEqual(loaded.last_processed_commit, "commit-1") + + def test_load_missing_repository(self): + store = CheckpointStore() + self.assertIsNone(store.load("repo-b")) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/diff_normalizer_test.py b/application/tests/harvester_test/diff_normalizer_test.py new file mode 100644 index 000000000..04eb1ce3f --- /dev/null +++ b/application/tests/harvester_test/diff_normalizer_test.py @@ -0,0 +1,117 @@ +import unittest +from datetime import datetime + +from application.utils.harvester.diff_normalizer import ( + DiffNormalizer, +) + +from application.utils.harvester.models import ( + DiffBlock, +) + + +DIFF_METADATA = { + "repository": "OWASP/ASVS", + "commit_sha": "abc123", + "committed_at": datetime(2026, 1, 1), +} + + +class DiffNormalizerTests(unittest.TestCase): + def test_whitespace_normalization(self): + normalizer = DiffNormalizer() + + blocks = [ + DiffBlock( + file_path="README.md", + added_lines=[ + " Hello World ", + "\t\tTabs\t\tEverywhere\t", + "", + " ", + "Unicode\u00a0Space", + "Mix\t of\t tabs and spaces", + " Multiple words together ", + "\u00a0\u00a0Leading unicode spaces\u00a0", + " ## Authentication ", + " - Use MFA ", + " `inline code` ", + " **Important** ", + ], + **DIFF_METADATA, + ) + ] + + result = normalizer.normalize(blocks) + + self.assertEqual( + result[0].added_lines, + [ + "Hello World", + "Tabs Everywhere", + "Unicode Space", + "Mix of tabs and spaces", + "Multiple words together", + "Leading unicode spaces", + "## Authentication", + "- Use MFA", + "`inline code`", + "**Important**", + ], + ) + + def test_remove_empty_lines(self): + normalizer = DiffNormalizer() + + blocks = [ + DiffBlock( + file_path="README.md", + added_lines=[ + "", + " ", + "Hello", + ], + **DIFF_METADATA, + ) + ] + + result = normalizer.normalize(blocks) + + self.assertEqual( + result[0].added_lines, + [ + "Hello", + ], + ) + + def test_multiple_blocks(self): + normalizer = DiffNormalizer() + + blocks = [ + DiffBlock( + file_path="a.md", + added_lines=[" One "], + **DIFF_METADATA, + ), + DiffBlock( + file_path="b.md", + added_lines=[" Two "], + **DIFF_METADATA, + ), + ] + + result = normalizer.normalize(blocks) + + self.assertEqual( + result[0].added_lines, + ["One"], + ) + + self.assertEqual( + result[1].added_lines, + ["Two"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/diff_parser_test.py b/application/tests/harvester_test/diff_parser_test.py new file mode 100644 index 000000000..a4444e8d9 --- /dev/null +++ b/application/tests/harvester_test/diff_parser_test.py @@ -0,0 +1,103 @@ +from datetime import UTC, datetime +import unittest + +from application.utils.harvester.diff_parser import ( + DiffParser, +) + +TEST_REPOSITORY = "OWASP/ASVS" +TEST_COMMIT_SHA = "abc123" +TEST_COMMITTED_AT = datetime.now(UTC) + + +class DiffParserTests(unittest.TestCase): + def test_single_file_diff(self): + parser = DiffParser() + + diff = """diff --git a/test.md b/test.md +--- a/test.md ++++ b/test.md +@@ +-old ++new ++another +""" + + blocks = parser.parse( + diff, + repository=TEST_REPOSITORY, + commit_sha=TEST_COMMIT_SHA, + committed_at=TEST_COMMITTED_AT, + ) + + self.assertEqual(len(blocks), 1) + + self.assertEqual( + blocks[0].file_path, + "test.md", + ) + + self.assertEqual( + blocks[0].added_lines, + [ + "new", + "another", + ], + ) + + self.assertEqual(blocks[0].repository, TEST_REPOSITORY) + self.assertEqual(blocks[0].commit_sha, TEST_COMMIT_SHA) + self.assertEqual(blocks[0].committed_at, TEST_COMMITTED_AT) + + def test_multiple_files(self): + parser = DiffParser() + + diff = """diff --git a/a.md b/a.md +@@ ++one +diff --git a/b.md b/b.md +@@ ++two +""" + + blocks = parser.parse( + diff, + repository=TEST_REPOSITORY, + commit_sha=TEST_COMMIT_SHA, + committed_at=TEST_COMMITTED_AT, + ) + + self.assertEqual(len(blocks), 2) + + self.assertEqual(blocks[0].file_path, "a.md") + self.assertEqual(blocks[1].file_path, "b.md") + + self.assertEqual(blocks[0].repository, TEST_REPOSITORY) + self.assertEqual(blocks[1].repository, TEST_REPOSITORY) + + def test_deleted_lines_are_ignored(self): + parser = DiffParser() + + diff = """diff --git a/test.md b/test.md +@@ +-old ++new +""" + + blocks = parser.parse( + diff, + repository=TEST_REPOSITORY, + commit_sha=TEST_COMMIT_SHA, + committed_at=TEST_COMMITTED_AT, + ) + + self.assertEqual( + blocks[0].added_lines, + [ + "new", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/diff_pipeline_test.py b/application/tests/harvester_test/diff_pipeline_test.py new file mode 100644 index 000000000..07160f170 --- /dev/null +++ b/application/tests/harvester_test/diff_pipeline_test.py @@ -0,0 +1,73 @@ +from datetime import UTC, datetime +import subprocess +import time +import unittest +import os + +from application.utils.harvester.diff_normalizer import DiffNormalizer +from application.utils.harvester.diff_parser import DiffParser +from application.utils.harvester.diff_retriever import DiffRetriever +from application.utils.harvester.git_repository_client import GitRepositoryClient + + +class DiffPipelineBenchmark(unittest.TestCase): + """ + Simple benchmark to ensure the complete diff pipeline remains fast. + + This is not intended as a strict performance benchmark, only as a + regression guard against accidental slowdowns. + """ + + def test_pipeline_benchmark(self): + + if os.getenv("OPENCRE_RUN_NETWORK_TESTS") != "1": + self.skipTest("Network benchmark disabled") + + client = GitRepositoryClient( + "OWASP", + "ASVS", + "master", + ) + client.sync() + + head_commit = client.get_current_commit_sha() + + previous_commit = subprocess.run( + [ + "git", + "-C", + str(client.get_local_path()), + "rev-parse", + "HEAD~1", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ).stdout.strip() + + retriever = DiffRetriever(client) + parser = DiffParser() + normalizer = DiffNormalizer() + + start = time.perf_counter() + + diff = retriever.get_diff( + previous_commit, + head_commit, + ) + + blocks = parser.parse( + diff, + repository="OWASP/ASVS", + commit_sha=head_commit, + committed_at=datetime.now(UTC), + ) + + normalizer.normalize(blocks) + + elapsed = time.perf_counter() - start + + print(f"\nPipeline took {elapsed:.3f}s") + + self.assertLess(elapsed, 5) diff --git a/application/tests/harvester_test/diff_retriever_test.py b/application/tests/harvester_test/diff_retriever_test.py new file mode 100644 index 000000000..502ac2d39 --- /dev/null +++ b/application/tests/harvester_test/diff_retriever_test.py @@ -0,0 +1,99 @@ +import unittest +from unittest.mock import MagicMock +from unittest.mock import patch +from unittest.mock import call + +from application.utils.harvester.diff_retriever import ( + DiffRetriever, +) + + +class DiffRetrieverTests(unittest.TestCase): + @patch("application.utils.harvester.diff_retriever.subprocess.run") + def test_get_diff(self, mock_run): + mock_run.side_effect = [ + MagicMock(stdout="abc123\n"), + MagicMock(stdout="def456\n"), + MagicMock(stdout=b"diff --git a/README.md b/README.md\n"), + ] + + client = MagicMock() + client.get_local_path.return_value = "/tmp/repo" + + retriever = DiffRetriever(client) + + diff = retriever.get_diff( + "abc123", + "def456", + ) + + self.assertEqual( + diff, + "diff --git a/README.md b/README.md\n", + ) + + mock_run.assert_has_calls( + [ + call( + [ + "git", + "-C", + "/tmp/repo", + "rev-parse", + "--verify", + "--end-of-options", + "abc123^{commit}", + ], + check=True, + capture_output=True, + text=True, + timeout=60, + ), + call( + [ + "git", + "-C", + "/tmp/repo", + "rev-parse", + "--verify", + "--end-of-options", + "def456^{commit}", + ], + check=True, + capture_output=True, + text=True, + timeout=60, + ), + call( + [ + "git", + "-C", + "/tmp/repo", + "diff", + "abc123", + "def456", + ], + check=True, + capture_output=True, + timeout=300, + ), + ] + ) + + @patch("application.utils.harvester.diff_retriever.subprocess.run") + def test_large_diff_raises(self, mock_run): + mock_run.return_value = MagicMock( + stdout=b"A" * (51 * 1024 * 1024), + ) + + client = MagicMock() + client.get_local_path.return_value = "/tmp/repo" + + retriever = DiffRetriever(client) + + with self.assertRaises(ValueError): + retriever.get_diff("a", "b") + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/file_filter_test.py b/application/tests/harvester_test/file_filter_test.py new file mode 100644 index 000000000..82412df0d --- /dev/null +++ b/application/tests/harvester_test/file_filter_test.py @@ -0,0 +1,121 @@ +import unittest + +from application.utils.harvester.file_filter import ( + FileFilter, +) + + +class FileFilterTests(unittest.TestCase): + def test_extension_filtering(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + "README.md", + "image.png", + "script.js", + ] + ) + + self.assertEqual( + result, + ["README.md"], + ) + + def test_path_exclusion(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + ".github/workflows/test.yml", + "docs/setup.md", + ] + ) + + self.assertEqual( + result, + ["docs/setup.md"], + ) + + def test_combined_filtering(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + "README.md", + ".github/workflows/README.md", + "node_modules/react/README.md", + "docs/setup.md", + ] + ) + + self.assertEqual( + result, + [ + "README.md", + "docs/setup.md", + ], + ) + + def test_empty_overrides_are_respected(self): + file_filter = FileFilter( + exclude_patterns=[], + allowed_extensions=set(), + ) + + result = file_filter.filter_files( + [ + "README.md", + "image.png", + ] + ) + + self.assertEqual(result, []) + + def test_nested_directory_globs(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + ".github/README.md", + "packages/site/node_modules/README.md", + "docs/archive/old.md", + ".cursor/rules/project.md", + "docs/setup.md", + ] + ) + + self.assertEqual(result, ["docs/setup.md"]) + + def test_explicit_empty_exclusions(self): + file_filter = FileFilter(exclude_patterns=[]) + + result = file_filter.filter_files( + [ + ".github/README.md", + ] + ) + + self.assertEqual( + result, + [".github/README.md"], + ) + + def test_default_instance_isolation(self): + first = FileFilter() + second = FileFilter() + + first.exclude_patterns.append("**/foo/**") + + self.assertNotIn( + "**/foo/**", + second.exclude_patterns, + ) + + def test_empty_extension_raises(self): + with self.assertRaises(ValueError): + FileFilter(allowed_extensions={""}) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/filtering_benchmark_test.py b/application/tests/harvester_test/filtering_benchmark_test.py new file mode 100644 index 000000000..a75c435ec --- /dev/null +++ b/application/tests/harvester_test/filtering_benchmark_test.py @@ -0,0 +1,30 @@ +import unittest + +from application.utils.harvester.file_filter import FileFilter +from application.utils.harvester.filtering_benchmark import FilteringBenchmark + + +class FilteringBenchmarkTests(unittest.TestCase): + def test_filtering_benchmark(self): + files = [ + "README.md", + ".github/workflows/ci.yml", + "docs/guide.md", + "image.png", + "notes.txt", + "package-lock.json", + ] + + benchmark = FilteringBenchmark(file_filter=FileFilter()) + + result = benchmark.run(files) + + self.assertEqual(result.total_files, 6) + self.assertEqual(result.retained_files, 3) + self.assertEqual(result.filtered_files, 3) + self.assertEqual(result.retention_rate, 0.5) + self.assertEqual(result.filtering_rate, 0.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/filtering_metrics_test.py b/application/tests/harvester_test/filtering_metrics_test.py new file mode 100644 index 000000000..252cb0941 --- /dev/null +++ b/application/tests/harvester_test/filtering_metrics_test.py @@ -0,0 +1,35 @@ +import unittest + +from application.utils.harvester.filtering_metrics import ( + FilteringMetricsCollector, +) + + +class FilteringMetricsCollectorTests(unittest.TestCase): + def test_filtering_metrics_collection(self): + collector = FilteringMetricsCollector() + + collector.record_retained() + collector.record_retained() + collector.record_filtered() + + metrics = collector.build() + + self.assertEqual( + metrics.total_files, + 3, + ) + + self.assertEqual( + metrics.retained_files, + 2, + ) + + self.assertEqual( + metrics.filtered_files, + 1, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/git_repository_client_integration_test.py b/application/tests/harvester_test/git_repository_client_integration_test.py index 4803e1d67..f2f585098 100644 --- a/application/tests/harvester_test/git_repository_client_integration_test.py +++ b/application/tests/harvester_test/git_repository_client_integration_test.py @@ -7,6 +7,7 @@ from application.utils.harvester.git_repository_client import ( GitRepositoryClient, ) +from application.utils.harvester.change_detector import ChangeDetector class IntegrationGitRepositoryClient(GitRepositoryClient): @@ -201,6 +202,71 @@ def run_sync(client): self.assertEqual((self.cache / "test.txt").read_text(), "v1") + def test_change_detector_uses_captured_target_sha(self): + client = self.create_client() + client.clone() + + detector = ChangeDetector(client) + + base = client.get_current_commit_sha() + + (self.work / "file.txt").write_text("B") + git("add", ".", cwd=self.work) + git("commit", "-m", "second", cwd=self.work) + git("push", "origin", "main", cwd=self.work) + + client.fetch() + + target = client.get_current_commit_sha() + + (self.work / "another.txt").write_text("C") + git("add", ".", cwd=self.work) + git("commit", "-m", "third", cwd=self.work) + git("push", "origin", "main", cwd=self.work) + + client.fetch() + files = detector.get_modified_files_since(base, target) + commits = detector.get_commits_since(base, target) + + self.assertEqual(files, ["file.txt"]) + self.assertEqual(commits, [target]) + + def test_change_detector_returns_commits_oldest_first(self): + client = self.create_client() + client.clone() + + detector = ChangeDetector(client) + base = client.get_current_commit_sha() + + (self.work / "file.txt").write_text("B") + git("add", ".", cwd=self.work) + git("commit", "-m", "B", cwd=self.work) + commit_b = git_output("rev-parse", "HEAD", cwd=self.work) + + (self.work / "file.txt").write_text("C") + git("add", ".", cwd=self.work) + git("commit", "-m", "C", cwd=self.work) + commit_c = git_output("rev-parse", "HEAD", cwd=self.work) + + (self.work / "file.txt").write_text("D") + git("add", ".", cwd=self.work) + git("commit", "-m", "D", cwd=self.work) + commit_d = git_output("rev-parse", "HEAD", cwd=self.work) + + git("push", "origin", "main", cwd=self.work) + + client.fetch() + + commits = detector.get_commits_since(base, commit_d) + self.assertEqual( + commits, + [ + commit_b, + commit_c, + commit_d, + ], + ) + if __name__ == "__main__": unittest.main() diff --git a/application/tests/harvester_test/git_repository_client_test.py b/application/tests/harvester_test/git_repository_client_test.py index 774715617..8bbff6ab8 100644 --- a/application/tests/harvester_test/git_repository_client_test.py +++ b/application/tests/harvester_test/git_repository_client_test.py @@ -70,7 +70,8 @@ def test_sync_clones_when_repository_missing(self): mock_clone.assert_called_once() - def test_sync_fetches_when_repository_exists(self): + @patch("application.utils.harvester.git_repository_client.subprocess.run") + def test_sync_fetches_when_repository_exists(self, mock_run): client = GitRepositoryClient( owner="OWASP", repository="ASVS", @@ -114,6 +115,7 @@ def test_checkout_runs_git_command(self, mock_run): "-C", str(client.get_local_path()), "checkout", + "--", "main", ], check=True, diff --git a/application/tests/import_run_test.py b/application/tests/import_run_test.py index c3e303625..7b03161a6 100644 --- a/application/tests/import_run_test.py +++ b/application/tests/import_run_test.py @@ -1,6 +1,9 @@ """Tests for import run metadata (Step 6).""" +import json import unittest +from datetime import datetime, timezone + from application import create_app, sqla from application.database import db @@ -31,3 +34,52 @@ def test_get_latest_import_run(self) -> None: self.assertIsNotNone(latest) self.assertEqual(latest.id, run2.id) self.assertEqual(latest.version, "2.0") + + def test_create_artifact_ingest_event_and_chunk(self) -> None: + run = db.create_import_run(source="artifact_ingest", version="1.0") + observed_at = datetime.now(timezone.utc) + + event = db.create_artifact_ingest_event( + run_id=run.id, + artifact_id="artifact-1", + harvest_mode="backfill", + event_type="discovered", + source_json={"uri": "https://example.com/source"}, + locator_json={"path": "/tmp/source"}, + artifact_json={"id": "artifact-1"}, + harvest_json={"status": "ok"}, + observed_at=observed_at, + ) + + self.assertIsNotNone(event.id) + self.assertEqual(event.run_id, run.id) + self.assertEqual(event.artifact_id, "artifact-1") + self.assertEqual( + json.loads(event.source_json), {"uri": "https://example.com/source"} + ) + self.assertEqual(json.loads(event.locator_json), {"path": "/tmp/source"}) + self.assertEqual(json.loads(event.artifact_json), {"id": "artifact-1"}) + self.assertEqual(json.loads(event.harvest_json), {"status": "ok"}) + self.assertEqual( + event.observed_at.replace(tzinfo=None), + observed_at.astimezone(timezone.utc).replace(tzinfo=None), + ) + self.assertIsNotNone(event.created_at) + + chunk = db.create_ingest_chunk( + artifact_event_id=event.id, + chunk_id="chunk-1", + text="hello world", + char_count=11, + span_json={"start": 0, "end": 11}, + delta_json={"op": "add"}, + ) + + self.assertIsNotNone(chunk.id) + self.assertEqual(chunk.artifact_event_id, event.id) + self.assertEqual(chunk.chunk_id, "chunk-1") + self.assertEqual(chunk.text, "hello world") + self.assertEqual(chunk.char_count, 11) + self.assertEqual(json.loads(chunk.span_json), {"start": 0, "end": 11}) + self.assertEqual(json.loads(chunk.delta_json), {"op": "add"}) + self.assertIsNotNone(chunk.created_at) diff --git a/application/utils/harvester/__init__.py b/application/utils/harvester/__init__.py index 2ac608b3e..9961aae16 100644 --- a/application/utils/harvester/__init__.py +++ b/application/utils/harvester/__init__.py @@ -17,12 +17,25 @@ from .git_repository_client import GitRepositoryClient from .repository_client import RepositoryClient from .repository_cache import build_repository_cache_path +from .file_filter import FileFilter +from .filtering_metrics import FilteringMetricsCollector +from .diff_retriever import DiffRetriever + +from .filtering_benchmark import ( + FilteringBenchmark, + FilteringBenchmarkResult, +) __all__ = [ "build_repository_cache_path", "ChunkingConfig", "ConfigLoaderError", + "DiffRetriever", "GitRepositoryClient", + "FileFilter", + "FilteringMetricsCollector", + "FilteringBenchmark", + "FilteringBenchmarkResult", "PathRules", "PollingConfig", "RepositoryClient", diff --git a/application/utils/harvester/change_detector.py b/application/utils/harvester/change_detector.py new file mode 100644 index 000000000..584d43d9c --- /dev/null +++ b/application/utils/harvester/change_detector.py @@ -0,0 +1,107 @@ +import logging +import subprocess + +from .git_repository_client import GitRepositoryClient + +logger = logging.getLogger(__name__) + + +class ChangeDetector: + def __init__(self, repository_client: GitRepositoryClient): + self.repository_client = repository_client + + def _resolve_commit(self, commit_sha: str) -> str: + try: + result = subprocess.run( + [ + "git", + "-C", + str(self.repository_client.get_local_path()), + "rev-parse", + "--verify", + "--end-of-options", + f"{commit_sha}^{{commit}}", + ], + capture_output=True, + text=True, + check=True, + timeout=60, + ) + except subprocess.CalledProcessError as exc: + logger.error("Git command failed: %s", exc.stderr) + raise + + return result.stdout.strip() + + def get_modified_files_since( + self, base_commit: str, target_commit: str + ) -> list[str]: + logger.info( + "Detecting changes between %s and %s", + base_commit, + target_commit, + ) + + base = self._resolve_commit(base_commit) + target = self._resolve_commit(target_commit) + + try: + result = subprocess.run( + [ + "git", + "-C", + str(self.repository_client.get_local_path()), + "diff", + "--name-only", + base, + target, + ], + capture_output=True, + text=True, + check=True, + timeout=60, + ) + except subprocess.CalledProcessError as exc: + logger.error("Git command failed: %s", exc.stderr) + raise + + files = [ + file_path for file_path in result.stdout.splitlines() if file_path.strip() + ] + + return sorted(set(files)) + + def get_commits_since(self, base_commit: str, target_commit: str) -> list[str]: + base = self._resolve_commit(base_commit) + target = self._resolve_commit(target_commit) + + try: + result = subprocess.run( + [ + "git", + "-C", + str(self.repository_client.get_local_path()), + "log", + "--reverse", + "--format=%H", + f"{base}..{target}", + ], + capture_output=True, + text=True, + check=True, + timeout=60, + ) + except subprocess.CalledProcessError as exc: + logger.error("Git command failed: %s", exc.stderr) + raise + + commits = [sha for sha in result.stdout.splitlines() if sha.strip()] + + logger.info( + "Detected %s commits between %s and %s", + len(commits), + base_commit, + target_commit, + ) + + return commits diff --git a/application/utils/harvester/checkpoint_store.py b/application/utils/harvester/checkpoint_store.py new file mode 100644 index 000000000..ed757bb56 --- /dev/null +++ b/application/utils/harvester/checkpoint_store.py @@ -0,0 +1,98 @@ +from typing import Any + +from sqlalchemy.exc import IntegrityError + +from application import sqla +from application.database.db import HarvesterCheckpoint +from .models import RepositoryCheckpoint + + +class CheckpointStore: + def __init__(self, session: Any = None) -> None: + self._session = session + + @property + def session(self) -> Any: + return self._session if self._session is not None else sqla.session + + def load(self, repository_id: str) -> RepositoryCheckpoint | None: + session = self.session + record = ( + session.query(HarvesterCheckpoint) + .filter_by(repository_id=repository_id) + .first() + ) + + if record is None: + return None + + return RepositoryCheckpoint( + repository_id=record.repository_id, + last_processed_commit=record.last_processed_commit, + updated_at=record.updated_at, + provider=record.provider, + owner=record.owner, + repository=record.repository, + branch=record.branch, + ) + + def save(self, checkpoint: RepositoryCheckpoint) -> None: + session = self.session + existing = ( + session.query(HarvesterCheckpoint) + .filter_by(repository_id=checkpoint.repository_id) + .first() + ) + + if existing is None: + canonical_conflict = ( + session.query(HarvesterCheckpoint) + .filter_by( + provider=checkpoint.provider, + owner=checkpoint.owner, + repository=checkpoint.repository, + branch=checkpoint.branch, + ) + .first() + ) + + if canonical_conflict is not None: + session.rollback() + raise ValueError("duplicate canonical source identity") + + new_record = HarvesterCheckpoint( + repository_id=checkpoint.repository_id, + provider=checkpoint.provider, + owner=checkpoint.owner, + repository=checkpoint.repository, + branch=checkpoint.branch, + last_processed_commit=checkpoint.last_processed_commit, + updated_at=checkpoint.updated_at, + ) + session.add(new_record) + try: + session.commit() + except IntegrityError: + session.rollback() + raise ValueError("duplicate canonical source identity") + except Exception: + session.rollback() + raise + return + + if ( + existing.provider != checkpoint.provider + or existing.owner != checkpoint.owner + or existing.repository != checkpoint.repository + or existing.branch != checkpoint.branch + ): + session.rollback() + raise ValueError("immutable repository identity") + + existing.last_processed_commit = checkpoint.last_processed_commit + existing.updated_at = checkpoint.updated_at + try: + session.commit() + except Exception: + session.rollback() + raise diff --git a/application/utils/harvester/diff_normalizer.py b/application/utils/harvester/diff_normalizer.py new file mode 100644 index 000000000..fe8773349 --- /dev/null +++ b/application/utils/harvester/diff_normalizer.py @@ -0,0 +1,47 @@ +import textacy.preprocessing as prep + +from application.utils.harvester import repository_client +from .models import DiffBlock + + +class DiffNormalizer: + """ + Normalizes extracted diff content. + + Whitespace is collapsed, Unicode normalized, + and empty lines removed. + """ + + def normalize_line(self, line: str) -> str: + line = prep.normalize.unicode(line) + line = prep.normalize.whitespace(line) + return line.strip() + + def normalize(self, blocks: list[DiffBlock]) -> list[DiffBlock]: + """ + Normalize every added line in each DiffBlock. + """ + normalized: list[DiffBlock] = [] + + for block in blocks: + cleaned_lines: list[str] = [] + + for line in block.added_lines: + line = self.normalize_line(line) + + if not line: + continue + + cleaned_lines.append(line) + + normalized.append( + DiffBlock( + file_path=block.file_path, + added_lines=cleaned_lines, + repository=block.repository, + commit_sha=block.commit_sha, + committed_at=block.committed_at, + ) + ) + + return normalized diff --git a/application/utils/harvester/diff_parser.py b/application/utils/harvester/diff_parser.py new file mode 100644 index 000000000..12b14784c --- /dev/null +++ b/application/utils/harvester/diff_parser.py @@ -0,0 +1,69 @@ +from datetime import datetime +import re + +from .models import DiffBlock + + +class DiffParser: + """ + Parses unified git diffs into DiffBlock objects. + + Only added lines are extracted. + Deleted lines and diff metadata are ignored. + """ + + def parse( + self, diff: str, repository: str, commit_sha: str, committed_at: datetime + ) -> list[DiffBlock]: + """ + Convert a unified git diff into DiffBlock objects. + """ + blocks: list[DiffBlock] = [] + + current_file: str | None = None + added_lines: list[str] = [] + + for line in diff.splitlines(): + if line.startswith("diff --git"): + if current_file is not None: + blocks.append( + DiffBlock( + file_path=current_file, + added_lines=added_lines, + repository=repository, + commit_sha=commit_sha, + committed_at=committed_at, + ) + ) + + match = re.match(r"diff --git a/(.+?) b/", line) + + current_file = match.group(1) if match else None + added_lines = [] + + continue + + if line.startswith("+++ b/") or line.startswith("+++ /dev/null"): + continue + + if line.startswith("--- a/") or line.startswith("--- /dev/null"): + continue + + if line.startswith("@@"): + continue + + if line.startswith("+"): + added_lines.append(line[1:]) + + if current_file is not None: + blocks.append( + DiffBlock( + file_path=current_file, + added_lines=added_lines, + repository=repository, + commit_sha=commit_sha, + committed_at=committed_at, + ) + ) + + return blocks diff --git a/application/utils/harvester/diff_retriever.py b/application/utils/harvester/diff_retriever.py new file mode 100644 index 000000000..f40f1cbdf --- /dev/null +++ b/application/utils/harvester/diff_retriever.py @@ -0,0 +1,99 @@ +import logging +import subprocess + +from .git_repository_client import GitRepositoryClient + +logger = logging.getLogger(__name__) + + +class DiffRetriever: + MAX_DIFF_SIZE_BYTES = 50 * 1024 * 1024 + """ + + Retrieves unified git diffs between two commits. + + This class is responsible only for retrieving raw diff text. + + Parsing and normalization are handled by downstream components. + + """ + + def __init__(self, repository_client: GitRepositoryClient) -> None: + self.repository_client = repository_client + + def get_diff(self, base_commit: str, target_commit: str = "HEAD") -> str: + """ + Return the unified git diff between two commits. + + Args: + base_commit: + Base commit SHA. + target_commit: + Target commit SHA or branch. + + Raises: + subprocess.CalledProcessError: + If git diff fails. + + ValueError: + If the diff exceeds the configured size limit. + """ + logger.info( + "Retrieving diff between %s and %s", + base_commit, + target_commit, + ) + + base_commit = self._resolve_commit(base_commit) + target_commit = self._resolve_commit(target_commit) + + try: + result = subprocess.run( + [ + "git", + "-C", + str(self.repository_client.get_local_path()), + "diff", + base_commit, + target_commit, + ], + check=True, + capture_output=True, + timeout=300, + ) + except subprocess.CalledProcessError as exc: + logger.error( + "Failed to retrieve diff: %s", + exc.stderr.decode("utf-8", errors="replace"), + ) + raise + + diff_bytes = result.stdout + + diff_size = len(diff_bytes) + + if diff_size > self.MAX_DIFF_SIZE_BYTES: + raise ValueError( + f"Diff size ({diff_size} bytes) exceeds " + f"maximum supported size ({self.MAX_DIFF_SIZE_BYTES} bytes)." + ) + + return diff_bytes.decode("utf-8", errors="replace") + + def _resolve_commit(self, commit: str) -> str: + result = subprocess.run( + [ + "git", + "-C", + str(self.repository_client.get_local_path()), + "rev-parse", + "--verify", + "--end-of-options", + f"{commit}^{{commit}}", + ], + check=True, + capture_output=True, + text=True, + timeout=60, + ) + return result.stdout.strip() diff --git a/application/utils/harvester/exclude_patterns.txt b/application/utils/harvester/exclude_patterns.txt index 499850ae8..2b92e85fc 100644 --- a/application/utils/harvester/exclude_patterns.txt +++ b/application/utils/harvester/exclude_patterns.txt @@ -4,7 +4,8 @@ # to filter non-documentation files during harvesting. -**/.git/* +**/.github/** +**/.git/** **/node_modules/** **/__pycache__/** **/.claude/** diff --git a/application/utils/harvester/file_filter.py b/application/utils/harvester/file_filter.py new file mode 100644 index 000000000..f1da0ee79 --- /dev/null +++ b/application/utils/harvester/file_filter.py @@ -0,0 +1,84 @@ +from pathlib import PurePosixPath +from pathlib import Path +import pathspec + +DEFAULT_ALLOWED_EXTENSIONS = { + ".md", + ".mdx", + ".rst", + ".txt", + ".adoc", +} + +DEFAULT_EXCLUDE_PATTERNS = tuple( + line.strip() + for line in ( + Path(__file__) + .with_name("exclude_patterns.txt") + .read_text(encoding="utf-8") + .splitlines() + ) + if line.strip() and not line.lstrip().startswith("#") +) + + +class FileFilter: + def __init__( + self, + exclude_patterns: list[str] | None = None, + allowed_extensions: set[str] | None = None, + ): + self.exclude_patterns: list[str] = ( + list(DEFAULT_EXCLUDE_PATTERNS) + if exclude_patterns is None + else list(exclude_patterns) + ) + + self.allowed_extensions: set[str] = ( + set(DEFAULT_ALLOWED_EXTENSIONS) + if allowed_extensions is None + else set(allowed_extensions) + ) + + self._validate_patterns() + + try: + self._exclude_spec = pathspec.PathSpec.from_lines( + "gitignore", + self.exclude_patterns, + ) + except Exception as exc: + raise ValueError("Invalid exclude glob") from exc + + def _validate_patterns(self) -> None: + if any(not pattern for pattern in self.exclude_patterns): + raise ValueError("Exclude pattern cannot be empty") + + if any(not extension for extension in self.allowed_extensions): + raise ValueError("Allowed extension cannot be empty") + + def _normalize_path(self, file_path: str) -> str: + return PurePosixPath(file_path).as_posix() + + def is_excluded_by_pattern(self, file_path: str) -> bool: + normalized = self._normalize_path(file_path) + return self._exclude_spec.match_file(normalized) + + def is_allowed_extension(self, file_path: str) -> bool: + return any( + file_path.endswith(extension) for extension in self.allowed_extensions + ) + + def filter_files(self, files: list[str]) -> list[str]: + filtered = [] + + for file_path in files: + if self.is_excluded_by_pattern(file_path): + continue + + if not self.is_allowed_extension(file_path): + continue + + filtered.append(file_path) + + return filtered diff --git a/application/utils/harvester/filtering_benchmark.py b/application/utils/harvester/filtering_benchmark.py new file mode 100644 index 000000000..5de2e7b69 --- /dev/null +++ b/application/utils/harvester/filtering_benchmark.py @@ -0,0 +1,32 @@ +from dataclasses import dataclass + +from .file_filter import FileFilter + + +@dataclass +class FilteringBenchmarkResult: + total_files: int + retained_files: int + filtered_files: int + retention_rate: float + filtering_rate: float + + +class FilteringBenchmark: + def __init__(self, file_filter: FileFilter): + self.file_filter = file_filter + + def run(self, file_paths: list[str]) -> FilteringBenchmarkResult: + retained = self.file_filter.filter_files(file_paths) + + total = len(file_paths) + retained_count = len(retained) + filtered_count = total - retained_count + + return FilteringBenchmarkResult( + total_files=total, + retained_files=retained_count, + filtered_files=filtered_count, + retention_rate=(retained_count / total if total else 0.0), + filtering_rate=(filtered_count / total if total else 0.0), + ) diff --git a/application/utils/harvester/filtering_metrics.py b/application/utils/harvester/filtering_metrics.py new file mode 100644 index 000000000..d496c3e2a --- /dev/null +++ b/application/utils/harvester/filtering_metrics.py @@ -0,0 +1,23 @@ +from .models import FilteringMetrics + + +class FilteringMetricsCollector: + def __init__(self): + self.total_files = 0 + self.retained_files = 0 + self.filtered_files = 0 + + def record_retained(self) -> None: + self.total_files += 1 + self.retained_files += 1 + + def record_filtered(self) -> None: + self.total_files += 1 + self.filtered_files += 1 + + def build(self) -> FilteringMetrics: + return FilteringMetrics( + total_files=self.total_files, + retained_files=self.retained_files, + filtered_files=self.filtered_files, + ) diff --git a/application/utils/harvester/git_repository_client.py b/application/utils/harvester/git_repository_client.py index 1390e6606..468be2665 100644 --- a/application/utils/harvester/git_repository_client.py +++ b/application/utils/harvester/git_repository_client.py @@ -20,6 +20,8 @@ def __init__( branch: str = "main", local_path: Path | None = None, ) -> None: + if branch.startswith("-"): + raise ValueError("Invalid git branch") self.owner = owner self.repository = repository self.branch = branch @@ -160,6 +162,7 @@ def checkout(self, reference: str) -> None: "-C", str(self.local_path), "checkout", + "--", reference, ], check=True, diff --git a/application/utils/harvester/models.py b/application/utils/harvester/models.py new file mode 100644 index 000000000..0eca718c9 --- /dev/null +++ b/application/utils/harvester/models.py @@ -0,0 +1,41 @@ +from dataclasses import dataclass +from datetime import datetime +from pydantic import BaseModel + + +@dataclass(slots=True) +class RepositoryCheckpoint: + repository_id: str + last_processed_commit: str | None + updated_at: datetime + provider: str + owner: str + repository: str + branch: str + + +@dataclass(slots=True) +class RepositoryChangeSet: + repository_id: str + commit_sha: str + modified_files: list[str] + + +class FilteringMetrics(BaseModel): + total_files: int + retained_files: int + filtered_files: int + + +@dataclass(slots=True) +class DiffBlock: + """ + Intermediate representation of normalized additions + extracted from a repository diff. + """ + + file_path: str + added_lines: list[str] + repository: str + commit_sha: str + committed_at: datetime | None = None diff --git a/application/utils/harvester/repository_cache.py b/application/utils/harvester/repository_cache.py index 94b721a60..a8c9b632c 100644 --- a/application/utils/harvester/repository_cache.py +++ b/application/utils/harvester/repository_cache.py @@ -18,6 +18,9 @@ def build_repository_cache_path( if not _VALID_COMPONENT.fullmatch(repository): raise ValueError(f"Invalid repository name: {repository}") + if branch in {".", ".."}: + raise ValueError("Invalid branch name") + encoded_branch = quote(branch, safe="") candidate = CACHE_ROOT / owner.casefold() / repository.casefold() / encoded_branch diff --git a/application/utils/harvester/repository_lock.py b/application/utils/harvester/repository_lock.py index 9890e6e2e..a9779033b 100644 --- a/application/utils/harvester/repository_lock.py +++ b/application/utils/harvester/repository_lock.py @@ -14,7 +14,7 @@ def repository_lock(repository_path: Path): Acquire an exclusive inter-process lock for a repository cache path. """ - lock_path = repository_path.with_suffix(".lock") + lock_path = repository_path.parent / f"{repository_path.name}.lock" lock_path.parent.mkdir(parents=True, exist_ok=True) with lock_path.open("w") as lock_file: diff --git a/migrations/versions/6a9d0d62ef41_add_harvester_checkpoint_table.py b/migrations/versions/6a9d0d62ef41_add_harvester_checkpoint_table.py new file mode 100644 index 000000000..4a3ed2d54 --- /dev/null +++ b/migrations/versions/6a9d0d62ef41_add_harvester_checkpoint_table.py @@ -0,0 +1,41 @@ +"""add harvester_checkpoint table + +Revision ID: 6a9d0d62ef41 +Revises: 9f1a2b3c4d5e +Create Date: 2026-07-25 + +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "6a9d0d62ef41" +down_revision = "9f1a2b3c4d5e" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "harvester_checkpoint", + sa.Column("repository_id", sa.String(), primary_key=True), + sa.Column("provider", sa.String(), nullable=False), + sa.Column("owner", sa.String(), nullable=False), + sa.Column("repository", sa.String(), nullable=False), + sa.Column("branch", sa.String(), nullable=False), + sa.Column("last_processed_commit", sa.String(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.UniqueConstraint( + "provider", + "owner", + "repository", + "branch", + name="uq_harvester_checkpoint_canonical_source", + ), + ) + + +def downgrade(): + op.drop_table("harvester_checkpoint") diff --git a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py new file mode 100644 index 000000000..9fad7bed6 --- /dev/null +++ b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py @@ -0,0 +1,82 @@ +"""add artifact ingest event and chunk tables + +Revision ID: 9f1a2b3c4d5e +Revises: e1f2a3b4c5d6 +Create Date: 2026-07-23 + +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "9f1a2b3c4d5e" +down_revision = "c7d8e9f0a1b2" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "artifact_ingest_event", + sa.Column("id", sa.String(), primary_key=True), + sa.Column("run_id", sa.String(), nullable=False), + sa.Column("artifact_id", sa.String(), nullable=False), + sa.Column("harvest_mode", sa.String(), nullable=False), + sa.Column("event_type", sa.String(), nullable=False), + sa.Column("source_json", sa.Text(), nullable=False), + sa.Column("locator_json", sa.Text(), nullable=False), + sa.Column("artifact_json", sa.Text(), nullable=False), + sa.Column("harvest_json", sa.Text(), nullable=False), + sa.Column("observed_at", sa.DateTime(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["run_id"], + ["import_run.id"], + onupdate="CASCADE", + ondelete="CASCADE", + ), + ) + op.create_unique_constraint( + "uq_artifact_ingest_event_run_artifact", + "artifact_ingest_event", + ["run_id", "artifact_id"], + ) + + op.create_table( + "ingest_chunk", + sa.Column("id", sa.String(), primary_key=True), + sa.Column("artifact_event_id", sa.String(), nullable=False), + sa.Column("chunk_id", sa.String(), nullable=False), + sa.Column("text", sa.Text(), nullable=False), + sa.Column("char_count", sa.Integer(), nullable=False), + sa.Column("span_json", sa.Text(), nullable=False), + sa.Column("delta_json", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["artifact_event_id"], + ["artifact_ingest_event.id"], + onupdate="CASCADE", + ondelete="CASCADE", + ), + ) + op.create_unique_constraint( + "uq_ingest_chunk_artifact_chunk", + "ingest_chunk", + ["artifact_event_id", "chunk_id"], + ) + + +def downgrade(): + op.drop_constraint( + "uq_ingest_chunk_artifact_chunk", + "ingest_chunk", + type_="unique", + ) + op.drop_table("ingest_chunk") + op.drop_constraint( + "uq_artifact_ingest_event_run_artifact", + "artifact_ingest_event", + type_="unique", + ) + op.drop_table("artifact_ingest_event") diff --git a/requirements-dev.txt b/requirements-dev.txt index 4b8784bdc..283942164 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -78,6 +78,7 @@ types-PyYAML typing-inspect pycodestyle pyflakes +textacy # lint / test / typecheck black==24.4.2