diff --git a/amber/src/main/python/pytexera/storage/dataset_file_document.py b/amber/src/main/python/pytexera/storage/dataset_file_document.py index b8d778d3421..2d14f858e47 100644 --- a/amber/src/main/python/pytexera/storage/dataset_file_document.py +++ b/amber/src/main/python/pytexera/storage/dataset_file_document.py @@ -59,18 +59,18 @@ def __init__(self, file_path: str): :param file_path: Expected format - - "/datasets/ownerEmail/datasetName/versionName/fileRelativePath" + "/dataset/ownerEmail/datasetName/versionName/fileRelativePath" Example: - "/datasets/bob@texera.com/twitterDataset/v1/california/tw1.csv" + "/dataset/bob@texera.com/twitterDataset/v1/california/tw1.csv" """ parts = file_path.strip("/").split("/") invalid_format = ValueError( "Invalid file path format. Expected: " - "/datasets/ownerEmail/datasetName/versionName/fileRelativePath" + "/dataset/ownerEmail/datasetName/versionName/fileRelativePath" ) - # TODO(datasets-prefix): require the prefix once all stored paths are migrated (36.sql) and ml model support work is completed. + # TODO(dataset-prefix): require the prefix once all stored paths are migrated (36.sql) and ml model support work is completed. if parts and parts[0] in {t.value for t in ResourceType}: if len(parts) < 5: raise invalid_format @@ -80,7 +80,7 @@ def __init__(self, file_path: str): self.version_name = parts[3] self.file_relative_path = "/".join(parts[4:]) elif len(parts) >= 4: - self.resource_type = ResourceType.DATASETS + self.resource_type = ResourceType.DATASET self.owner_email = parts[0] self.dataset_name = parts[1] self.version_name = parts[2] diff --git a/amber/src/main/python/pytexera/storage/resource_type.py b/amber/src/main/python/pytexera/storage/resource_type.py index 596fb5d7453..e77ec9dd491 100644 --- a/amber/src/main/python/pytexera/storage/resource_type.py +++ b/amber/src/main/python/pytexera/storage/resource_type.py @@ -21,4 +21,4 @@ class ResourceType(str, Enum): """The leading segment of a logical file path, identifying the resource kind""" - DATASETS = "datasets" + DATASET = "dataset" diff --git a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/SearchQueryBuilder.scala b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/SearchQueryBuilder.scala index e19755e6116..ab4c9258341 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/SearchQueryBuilder.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/SearchQueryBuilder.scala @@ -19,6 +19,7 @@ package org.apache.texera.web.resource.dashboard +import org.apache.texera.amber.core.storage.ResourceType import org.apache.texera.dao.SqlServer import org.apache.texera.web.resource.dashboard.DashboardResource.{ DashboardClickableFileEntry, @@ -36,7 +37,7 @@ object SearchQueryBuilder { val FILE_RESOURCE_TYPE = "file" val WORKFLOW_RESOURCE_TYPE = "workflow" val PROJECT_RESOURCE_TYPE = "project" - val DATASET_RESOURCE_TYPE = "dataset" + val DATASET_RESOURCE_TYPE = ResourceType.Dataset.toString val ALL_RESOURCE_TYPE = "" } diff --git a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/hub/EntityType.scala b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/hub/EntityType.scala index 0cf1bd4cc8d..bd83b7837e9 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/hub/EntityType.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/hub/EntityType.scala @@ -20,6 +20,7 @@ package org.apache.texera.web.resource.dashboard.hub import com.fasterxml.jackson.annotation.{JsonCreator, JsonValue} +import org.apache.texera.amber.core.storage.ResourceType /** * Defines all supported entity types for Hub resources. @@ -34,7 +35,7 @@ sealed trait EntityType { object EntityType { case object Workflow extends EntityType { val value = "workflow" } - case object Dataset extends EntityType { val value = "dataset" } + case object Dataset extends EntityType { val value: String = ResourceType.Dataset.toString } private val values = Seq(Workflow, Dataset) diff --git a/amber/src/test/python/pytexera/storage/test_dataset_file_document.py b/amber/src/test/python/pytexera/storage/test_dataset_file_document.py index bef134bc744..b90dcce81ba 100644 --- a/amber/src/test/python/pytexera/storage/test_dataset_file_document.py +++ b/amber/src/test/python/pytexera/storage/test_dataset_file_document.py @@ -47,18 +47,18 @@ def make_response(status_code: int, body=None, content: bytes = b""): class TestDatasetFileDocumentInit: def test_parses_prefixed_path(self, auth_env): - doc = DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") + doc = DatasetFileDocument("/dataset/bob@x.com/ds/v1/file.csv") assert doc.owner_email == "bob@x.com" assert doc.dataset_name == "ds" assert doc.version_name == "v1" assert doc.file_relative_path == "file.csv" def test_joins_nested_relative_path_back_with_slashes(self, auth_env): - doc = DatasetFileDocument("/datasets/bob@x.com/ds/v1/a/b/c/file.csv") + doc = DatasetFileDocument("/dataset/bob@x.com/ds/v1/a/b/c/file.csv") assert doc.file_relative_path == "a/b/c/file.csv" def test_strips_leading_and_trailing_slashes_before_parsing(self, auth_env): - doc = DatasetFileDocument("///datasets/bob@x.com/ds/v1/file.csv///") + doc = DatasetFileDocument("///dataset/bob@x.com/ds/v1/file.csv///") assert doc.owner_email == "bob@x.com" assert doc.file_relative_path == "file.csv" @@ -80,7 +80,7 @@ def test_unknown_leading_segment_is_read_as_a_legacy_owner(self, auth_env): def test_rejects_prefixed_path_with_too_few_segments(self, auth_env): with pytest.raises(ValueError, match="Invalid file path format"): - DatasetFileDocument("/datasets/bob@x.com/ds/v1") + DatasetFileDocument("/dataset/bob@x.com/ds/v1") def test_rejects_legacy_path_with_too_few_segments(self, auth_env): with pytest.raises(ValueError, match="Invalid file path format"): @@ -92,29 +92,29 @@ def test_requires_jwt_token_in_environment(self, monkeypatch): "FILE_SERVICE_GET_DATASET_PRESIGNED_URL_ENDPOINT", CUSTOM_ENDPOINT ) with pytest.raises(ValueError, match="JWT token is required"): - DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") + DatasetFileDocument("/dataset/bob@x.com/ds/v1/file.csv") def test_treats_empty_jwt_as_missing(self, monkeypatch): # An empty string is falsy and should be rejected just like an unset var. monkeypatch.setenv("USER_JWT_TOKEN", "") with pytest.raises(ValueError, match="JWT token is required"): - DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") + DatasetFileDocument("/dataset/bob@x.com/ds/v1/file.csv") def test_falls_back_to_default_endpoint_when_env_missing(self, monkeypatch): monkeypatch.setenv("USER_JWT_TOKEN", "tok") monkeypatch.delenv( "FILE_SERVICE_GET_DATASET_PRESIGNED_URL_ENDPOINT", raising=False ) - doc = DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") + doc = DatasetFileDocument("/dataset/bob@x.com/ds/v1/file.csv") assert doc.presign_endpoint == DEFAULT_ENDPOINT def test_uses_explicit_endpoint_from_environment(self, auth_env): - doc = DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") + doc = DatasetFileDocument("/dataset/bob@x.com/ds/v1/file.csv") assert doc.presign_endpoint == CUSTOM_ENDPOINT class TestGetPresignedUrl: - def _make_doc(self, monkeypatch, path="/datasets/bob@x.com/ds/v1/file.csv"): + def _make_doc(self, monkeypatch, path="/dataset/bob@x.com/ds/v1/file.csv"): monkeypatch.setenv("USER_JWT_TOKEN", "test-jwt-token") monkeypatch.setenv( "FILE_SERVICE_GET_DATASET_PRESIGNED_URL_ENDPOINT", CUSTOM_ENDPOINT @@ -144,9 +144,7 @@ def test_sends_bearer_authorization_header_with_jwt(self, monkeypatch): def test_url_encodes_filepath_query_parameter(self, monkeypatch): # urllib.parse.quote keeps "/" as safe by default, but encodes "@" # and " " — pin both pieces so the contract is explicit. - doc = self._make_doc( - monkeypatch, path="/datasets/bob@x.com/ds/v1/data file.csv" - ) + doc = self._make_doc(monkeypatch, path="/dataset/bob@x.com/ds/v1/data file.csv") with patch( "pytexera.storage.dataset_file_document.requests.Session.get" ) as mock_get: @@ -158,16 +156,16 @@ def test_url_encodes_filepath_query_parameter(self, monkeypatch): assert "bob%40x.com" in file_path assert file_path.startswith("/") - def test_sends_datasets_prefixed_filepath(self, monkeypatch): - # The reconstructed filePath sent to the file-service carries the "datasets" prefix. - doc = self._make_doc(monkeypatch, path="/datasets/bob@x.com/ds/v1/file.csv") + def test_sends_dataset_prefixed_filepath(self, monkeypatch): + # The reconstructed filePath sent to the file-service carries the "dataset" prefix. + doc = self._make_doc(monkeypatch, path="/dataset/bob@x.com/ds/v1/file.csv") with patch( "pytexera.storage.dataset_file_document.requests.Session.get" ) as mock_get: mock_get.return_value = make_response(200, body={"presignedUrl": "u"}) doc.get_presigned_url() _, kwargs = mock_get.call_args - assert kwargs["params"]["filePath"].startswith("/datasets/") + assert kwargs["params"]["filePath"].startswith("/dataset/") def test_calls_configured_endpoint(self, monkeypatch): doc = self._make_doc(monkeypatch) @@ -235,7 +233,7 @@ def _make_doc(self, monkeypatch): monkeypatch.setenv( "FILE_SERVICE_GET_DATASET_PRESIGNED_URL_ENDPOINT", CUSTOM_ENDPOINT ) - return DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") + return DatasetFileDocument("/dataset/bob@x.com/ds/v1/file.csv") def test_returns_bytesio_with_downloaded_content(self, monkeypatch): doc = self._make_doc(monkeypatch) @@ -291,7 +289,7 @@ def _make_doc(self, monkeypatch): monkeypatch.setenv( "FILE_SERVICE_GET_DATASET_PRESIGNED_URL_ENDPOINT", CUSTOM_ENDPOINT ) - return DatasetFileDocument("/datasets/bob@x.com/ds/v1/file.csv") + return DatasetFileDocument("/dataset/bob@x.com/ds/v1/file.csv") def test_presigned_url_request_passes_request_timeout(self, monkeypatch): doc = self._make_doc(monkeypatch) diff --git a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala index a0e14bcc8dd..b623a56ed29 100644 --- a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala @@ -910,7 +910,7 @@ class WorkflowExecutionsResourceSpec val content = """{ | "operators": [ - | {"operatorID": "scanA", "operatorProperties": {"fileName": "/datasets/owner@example.com/LockedDS/v1/data.csv"}}, + | {"operatorID": "scanA", "operatorProperties": {"fileName": "/dataset/owner@example.com/LockedDS/v1/data.csv"}}, | {"operatorID": "downstreamB", "operatorProperties": {}} | ], | "links": [ @@ -969,7 +969,7 @@ class WorkflowExecutionsResourceSpec val content = """{ | "operators": [ - | {"operatorID": "scan", "operatorProperties": {"fileName": "/datasets/test@example.com/MyDS/v1/data.csv"}} + | {"operatorID": "scan", "operatorProperties": {"fileName": "/dataset/test@example.com/MyDS/v1/data.csv"}} | ], | "links": [] |}""".stripMargin @@ -1408,7 +1408,7 @@ class WorkflowExecutionsResourceSpec private def scanOperator(operatorId: String, datasetName: String): String = s"""{"operatorID": "$operatorId", "operatorProperties": """ + - s"""{"fileName": "/datasets/$foreignOwnerEmail/$datasetName/v1/data.csv"}}""" + s"""{"fileName": "/dataset/$foreignOwnerEmail/$datasetName/v1/data.csv"}}""" // Seeds three datasets owned by somebody other than testUser and wires the workflow as // diff --git a/bin/single-node/examples/workflows/[Example] Data Exploration on Movies Dataset.json b/bin/single-node/examples/workflows/[Example] Data Exploration on Movies Dataset.json index cdc9a0e8708..5df44f53828 100644 --- a/bin/single-node/examples/workflows/[Example] Data Exploration on Movies Dataset.json +++ b/bin/single-node/examples/workflows/[Example] Data Exploration on Movies Dataset.json @@ -8,7 +8,7 @@ "fileEncoding": "UTF_8", "customDelimiter": ",", "hasHeader": true, - "fileName": "/datasets/texera/popular-movies-of-imdb/v1/TMDb_updated.csv", + "fileName": "/dataset/texera/popular-movies-of-imdb/v1/TMDb_updated.csv", "offset": 0, "limit": 1000 }, diff --git a/bin/single-node/examples/workflows/[Example] Machine Learning on Iris Dataset.json b/bin/single-node/examples/workflows/[Example] Machine Learning on Iris Dataset.json index 7e9a5d11fd4..e4cdc3a8aff 100644 --- a/bin/single-node/examples/workflows/[Example] Machine Learning on Iris Dataset.json +++ b/bin/single-node/examples/workflows/[Example] Machine Learning on Iris Dataset.json @@ -8,7 +8,7 @@ "fileEncoding": "UTF_8", "customDelimiter": ",", "hasHeader": true, - "fileName": "/datasets/texera/iris-species/v1/Iris.csv" + "fileName": "/dataset/texera/iris-species/v1/Iris.csv" }, "inputPorts": [], "outputPorts": [ diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala index 1a2e4fcf2a9..95127e234d3 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala @@ -93,7 +93,7 @@ object FileResolver { * caller can dispatch to the right backing table. * - Legacy unprefixed (datasets only, backward compat): /ownerEmail/datasetName/versionName/ * fileRelativePath (>= 4 segments), resolved as a dataset. Models are new and always require - * the /models/ prefix. + * the /model/ prefix. * * @param fileName the file path to parse * @return Some((resourceType, ownerEmail, resourceName, versionName, fileRelativePath)) if valid, @@ -115,11 +115,11 @@ object FileResolver { ) case None => // Legacy unprefixed dataset path (backward compat): /ownerEmail/datasetName/versionName/. - // TODO(datasets-prefix): require the prefix once all stored paths are migrated (36.sql). + // TODO(dataset-prefix): require the prefix once all stored paths are migrated (36.sql). if (pathSegments.length >= 4) Some( ( - ResourceType.Datasets, + ResourceType.Dataset, pathSegments(0), pathSegments(1), pathSegments(2), @@ -139,8 +139,8 @@ object FileResolver { * * Input: //ownerEmail/resourceName/versionName/fileRelativePath (or legacy unprefixed) * Output: {scheme}:///{repositoryName}/{versionHash}/fileRelativePath - * e.g. /datasets/bob@x.com/twitter/v1/dir/f.csv -> dataset:///dataset-15/adeq233td/dir/f.csv - * /models/bob@x.com/resnet/v1/weights/m.pt -> model:///model-15/adeq233td/weights/m.pt + * e.g. /dataset/bob@x.com/twitter/v1/dir/f.csv -> dataset:///dataset-15/adeq233td/dir/f.csv + * /model/bob@x.com/resnet/v1/weights/m.pt -> model:///model-15/adeq233td/weights/m.pt * * @throws java.io.FileNotFoundException if the path is not a valid versioned-resource path, the * resource/version does not exist, or the URI is malformed @@ -152,10 +152,10 @@ object FileResolver { ) val (scheme, repositoryName, versionHash) = resourceType match { - case ResourceType.Datasets => + case ResourceType.Dataset => val (repo, hash) = lookupDataset(ownerEmail, resourceName, versionName, fileName) (DATASET_FILE_URI_SCHEME, repo, hash) - case ResourceType.Models => + case ResourceType.Model => val (repo, hash) = lookupModel(ownerEmail, resourceName, versionName, fileName) (MODEL_FILE_URI_SCHEME, repo, hash) case other => @@ -306,7 +306,7 @@ object FileResolver { return None } parsePrefixedPath(path).collect { - case (ResourceType.Datasets, ownerEmail, datasetName, _, _) => (ownerEmail, datasetName) + case (ResourceType.Dataset, ownerEmail, datasetName, _, _) => (ownerEmail, datasetName) } } } diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/ResourceType.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/ResourceType.scala index 087dc6ef87d..47e90a1818c 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/ResourceType.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/ResourceType.scala @@ -26,8 +26,8 @@ package org.apache.texera.amber.core.storage * Path shape: //ownerEmail/resourceName/versionName/fileRelativePath */ object ResourceType extends Enumeration { - val Datasets: Value = Value("datasets") - val Models: Value = Value("models") + val Dataset: Value = Value("dataset") + val Model: Value = Value("model") /** * Returns the resource type named by the given path segment, or None if it is not a known diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala index e4143712226..2649a3d860f 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/storage/FileResolverSpec.scala @@ -122,13 +122,13 @@ class FileResolverSpec private val localCsvFilePath = "common/workflow-core/src/test/resources/country_sales_small.csv" - private val datasetACsvFilePath = "/datasets/test_user@test.com/test_dataset/v2/directory/a.csv" + private val datasetACsvFilePath = "/dataset/test_user@test.com/test_dataset/v2/directory/a.csv" - private val dataset1TxtFilePath = "/datasets/test_user@test.com/test_dataset/v1/1.txt" + private val dataset1TxtFilePath = "/dataset/test_user@test.com/test_dataset/v1/1.txt" - private val modelWeightsFilePath = "/models/test_user@test.com/test_model/v2/weights/model.pt" + private val modelWeightsFilePath = "/model/test_user@test.com/test_model/v2/weights/model.pt" - private val modelReadmeFilePath = "/models/test_user@test.com/test_model/v1/README.md" + private val modelReadmeFilePath = "/model/test_user@test.com/test_model/v1/README.md" // Legacy unprefixed form — still resolvable as a dataset (backward compat). private val unprefixedDataset1TxtFilePath = "/test_user@test.com/test_dataset/v1/1.txt" @@ -137,11 +137,11 @@ class FileResolverSpec private val unknownResourceTypeFilePath = "/notAResourceType/test_user@test.com/test_dataset/v1/1.txt" - // A model name presented under the datasets prefix must not resolve as a dataset. - private val modelNameUnderDatasetPrefix = "/datasets/test_user@test.com/test_model/v1/README.md" + // A model name presented under the dataset prefix must not resolve as a dataset. + private val modelNameUnderDatasetPrefix = "/dataset/test_user@test.com/test_model/v1/README.md" - // A dataset name presented under the models prefix must not resolve as a model. - private val datasetNameUnderModelPrefix = "/models/test_user@test.com/test_dataset/v1/1.txt" + // A dataset name presented under the model prefix must not resolve as a model. + private val datasetNameUnderModelPrefix = "/model/test_user@test.com/test_dataset/v1/1.txt" override protected def beforeAll(): Unit = { initializeDBAndReplaceDSLContext() @@ -225,7 +225,7 @@ class FileResolverSpec "FileResolver" should "throw not found exception when a prefixed path has too few segments" in { assertThrows[FileNotFoundException] { - FileResolver.resolve("/datasets/test_user@test.com/test_dataset") + FileResolver.resolve("/dataset/test_user@test.com/test_dataset") } } @@ -254,12 +254,12 @@ class FileResolverSpec "parseDatasetOwnerAndName" should "extract owner email and dataset name from a valid path" in { assert( - FileResolver.parseDatasetOwnerAndName("/datasets/test_user@test.com/test_dataset/v1/1.txt") + FileResolver.parseDatasetOwnerAndName("/dataset/test_user@test.com/test_dataset/v1/1.txt") == Some(("test_user@test.com", "test_dataset")) ) // extra segments beyond the file-relative path are ignored assert( - FileResolver.parseDatasetOwnerAndName("/datasets/owner@x.com/ds/v2/directory/nested/a.csv") + FileResolver.parseDatasetOwnerAndName("/dataset/owner@x.com/ds/v2/directory/nested/a.csv") == Some(("owner@x.com", "ds")) ) } @@ -272,7 +272,7 @@ class FileResolverSpec } it should "return None when the prefixed path has too few segments" in { - assert(FileResolver.parseDatasetOwnerAndName("/datasets/owner@x.com/ds").isEmpty) + assert(FileResolver.parseDatasetOwnerAndName("/dataset/owner@x.com/ds").isEmpty) assert(FileResolver.parseDatasetOwnerAndName("owner/dataset").isEmpty) } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExec.scala index fb27257865d..26a6496a241 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExec.scala @@ -32,7 +32,7 @@ import org.apache.texera.dao.jooq.generated.tables.User.USER object FileListerSourceOpExec { /** - * Parses a dataset version path (/datasets/ownerEmail/datasetName/versionName) into its + * Parses a dataset version path (/dataset/ownerEmail/datasetName/versionName) into its * (resourceTypePrefix, ownerEmail, datasetName, versionName) components. * * @throws IllegalArgumentException if the path is not a well-formed dataset version path @@ -42,14 +42,14 @@ object FileListerSourceOpExec { ): (String, String, String, String) = { val segments = datasetVersionPath.split("/").filter(_.nonEmpty) val invalidPath = s"Invalid dataset version path '$datasetVersionPath'; " + - "expected /datasets/ownerEmail/datasetName/versionName" + "expected /dataset/ownerEmail/datasetName/versionName" if (segments.headOption.exists(ResourceType.isValidPrefix)) { require(segments.length >= 4, invalidPath) (segments(0), segments(1), segments(2), segments(3)) } else { require(segments.length >= 3, invalidPath) - (ResourceType.Datasets.toString, segments(0), segments(1), segments(2)) + (ResourceType.Dataset.toString, segments(0), segments(1), segments(2)) } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExecSpec.scala index 19fb7513d54..ec0ffd9a290 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExecSpec.scala @@ -23,10 +23,10 @@ import org.scalatest.flatspec.AnyFlatSpec class FileListerSourceOpExecSpec extends AnyFlatSpec { - "parseDatasetVersionPath" should "extract components from a datasets-prefixed path" in { + "parseDatasetVersionPath" should "extract components from a dataset-prefixed path" in { val (prefix, owner, name, version) = - FileListerSourceOpExec.parseDatasetVersionPath("/datasets/bob@texera.com/twitterDataset/v1") - assert(prefix == "datasets") + FileListerSourceOpExec.parseDatasetVersionPath("/dataset/bob@texera.com/twitterDataset/v1") + assert(prefix == "dataset") assert(owner == "bob@texera.com") assert(name == "twitterDataset") assert(version == "v1") @@ -34,7 +34,7 @@ class FileListerSourceOpExecSpec extends AnyFlatSpec { it should "work when the owner segment is a username without an '@'" in { val (prefix, owner, name, version) = - FileListerSourceOpExec.parseDatasetVersionPath("/datasets/texera/test-ds/v1") + FileListerSourceOpExec.parseDatasetVersionPath("/dataset/texera/test-ds/v1") assert(owner == "texera") assert(name == "test-ds") assert(version == "v1") @@ -42,8 +42,8 @@ class FileListerSourceOpExecSpec extends AnyFlatSpec { it should "ignore trailing slashes and extra segments" in { val (prefix, owner, name, version) = - FileListerSourceOpExec.parseDatasetVersionPath("/datasets/alice/ds/v2/extra/") - assert(prefix == "datasets") + FileListerSourceOpExec.parseDatasetVersionPath("/dataset/alice/ds/v2/extra/") + assert(prefix == "dataset") assert(owner == "alice") assert(name == "ds") assert(version == "v2") @@ -52,11 +52,11 @@ class FileListerSourceOpExecSpec extends AnyFlatSpec { "canonicalVersionPath" should "rebuild the prefixed version path from its components" in { assert( FileListerSourceOpExec.canonicalVersionPath( - "datasets", + "dataset", "bob@texera.com", "twitterDataset", "v1" - ) == "/datasets/bob@texera.com/twitterDataset/v1" + ) == "/dataset/bob@texera.com/twitterDataset/v1" ) } @@ -64,17 +64,17 @@ class FileListerSourceOpExecSpec extends AnyFlatSpec { // Emitted paths must be rooted at the parsed components, not the raw configured path: // a stray "extra" segment would otherwise leak into every emitted file path. val (prefix, owner, name, version) = - FileListerSourceOpExec.parseDatasetVersionPath("/datasets/alice/ds/v2/extra/") + FileListerSourceOpExec.parseDatasetVersionPath("/dataset/alice/ds/v2/extra/") assert( FileListerSourceOpExec.canonicalVersionPath(prefix, owner, name, version) - == "/datasets/alice/ds/v2" + == "/dataset/alice/ds/v2" ) } it should "still accept a legacy unprefixed path" in { val (prefix, owner, name, version) = FileListerSourceOpExec.parseDatasetVersionPath("/alice/ds/v1") - assert(prefix == "datasets") + assert(prefix == "dataset") assert(owner == "alice") assert(name == "ds") assert(version == "v1") @@ -85,7 +85,7 @@ class FileListerSourceOpExecSpec extends AnyFlatSpec { FileListerSourceOpExec.parseDatasetVersionPath("/alice/ds/v1") assert( FileListerSourceOpExec.canonicalVersionPath(prefix, owner, name, version) - == "/datasets/alice/ds/v1" + == "/dataset/alice/ds/v1" ) } @@ -99,7 +99,7 @@ class FileListerSourceOpExecSpec extends AnyFlatSpec { it should "reject a prefixed path with too few segments" in { assertThrows[IllegalArgumentException] { - FileListerSourceOpExec.parseDatasetVersionPath("/datasets/alice/ds") + FileListerSourceOpExec.parseDatasetVersionPath("/dataset/alice/ds") } } diff --git a/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala b/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala index 75391149c6c..e9914d51a8d 100644 --- a/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala +++ b/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala @@ -352,7 +352,7 @@ class DatasetResource extends LazyLogging { private val COVER_IMAGE_SIZE_LIMIT_BYTES: Long = 10 * 1024 * 1024 // 10 MB private val ALLOWED_IMAGE_EXTENSIONS: Set[String] = Set(".jpg", ".jpeg", ".png", ".gif", ".webp") - private val resourceType = ResourceType.Datasets + private val resourceType = ResourceType.Dataset /** * Helper function to get the dataset from DB with additional information including user access privilege and owner email diff --git a/file-service/src/main/scala/org/apache/texera/service/type/dataset/DatasetFileNode.scala b/file-service/src/main/scala/org/apache/texera/service/type/dataset/DatasetFileNode.scala index 821d02045c4..85709b8e5b7 100644 --- a/file-service/src/main/scala/org/apache/texera/service/type/dataset/DatasetFileNode.scala +++ b/file-service/src/main/scala/org/apache/texera/service/type/dataset/DatasetFileNode.scala @@ -25,8 +25,8 @@ import org.apache.texera.amber.core.storage.ResourceType import scala.collection.mutable // DatasetFileNode represents a unique file in dataset, its full path is in the format of: -// /datasets/ownerEmail/datasetName/versionName/fileRelativePath -// e.g. /datasets/bob@texera.com/twitterDataset/v1/california/irvine/tw1.csv +// /dataset/ownerEmail/datasetName/versionName/fileRelativePath +// e.g. /dataset/bob@texera.com/twitterDataset/v1/california/irvine/tw1.csv class DatasetFileNode( val name: String, // direct name of this node val nodeType: String, // "file" or "directory" @@ -79,10 +79,10 @@ object DatasetFileNode { ): List[DatasetFileNode] = { val rootNode = new DatasetFileNode("/", "directory", null, "") - // Root the tree at the datasets prefix node (a directory node named "datasets"). - val datasetsNode = - new DatasetFileNode(ResourceType.Datasets.toString, "directory", rootNode, "") - rootNode.children = Some(List(datasetsNode)) + // Root the tree at the dataset prefix node (a directory node named "dataset"). + val prefixNode = + new DatasetFileNode(ResourceType.Dataset.toString, "directory", rootNode, "") + rootNode.children = Some(List(prefixNode)) // Owner level nodes map val ownerNodes = mutable.Map[String, DatasetFileNode]() @@ -91,8 +91,8 @@ object DatasetFileNode { case ((ownerEmail, datasetName, versionName), objects) => val ownerNode = ownerNodes.getOrElseUpdate( ownerEmail, { - val newNode = new DatasetFileNode(ownerEmail, "directory", datasetsNode, ownerEmail) - datasetsNode.children = Some(datasetsNode.getChildren :+ newNode) + val newNode = new DatasetFileNode(ownerEmail, "directory", prefixNode, ownerEmail) + prefixNode.children = Some(prefixNode.getChildren :+ newNode) newNode } ) diff --git a/file-service/src/test/scala/org/apache/texera/service/type/dataset/DatasetFileNodeSpec.scala b/file-service/src/test/scala/org/apache/texera/service/type/dataset/DatasetFileNodeSpec.scala index 160412f8b93..12321d4180b 100644 --- a/file-service/src/test/scala/org/apache/texera/service/type/dataset/DatasetFileNodeSpec.scala +++ b/file-service/src/test/scala/org/apache/texera/service/type/dataset/DatasetFileNodeSpec.scala @@ -99,13 +99,13 @@ class DatasetFileNodeSpec extends AnyFlatSpec with Matchers { Map(("bob@texera.com", "twitter", "v1") -> objects) ) - // The tree is rooted at a single "datasets" prefix node; owners nest under it. + // The tree is rooted at a single "dataset" prefix node; owners nest under it. roots should have size 1 - val datasetsNode = roots.head - datasetsNode.getName shouldBe "datasets" - datasetsNode.getNodeType shouldBe "directory" + val prefixNode = roots.head + prefixNode.getName shouldBe "dataset" + prefixNode.getNodeType shouldBe "directory" - val ownerNode = datasetsNode.getChildren.find(_.getName == "bob@texera.com").get + val ownerNode = prefixNode.getChildren.find(_.getName == "bob@texera.com").get ownerNode.getNodeType shouldBe "directory" val datasetNode = ownerNode.getChildren.find(_.getName == "twitter").get @@ -123,7 +123,7 @@ class DatasetFileNodeSpec extends AnyFlatSpec with Matchers { val file1 = bDir.getChildren.find(_.getName == "1.csv").get file1.getNodeType shouldBe "file" file1.getSize shouldBe Some(2L) - file1.getFilePath shouldBe "/datasets/bob@texera.com/twitter/v1/b/1.csv" + file1.getFilePath shouldBe "/dataset/bob@texera.com/twitter/v1/b/1.csv" // Total size equals the sum of the three files. DatasetFileNode.calculateTotalSize(roots) shouldBe 6L diff --git a/frontend/src/app/common/type/datasetVersionFileTree.spec.ts b/frontend/src/app/common/type/datasetVersionFileTree.spec.ts index 64231dd1564..207182e5fd7 100644 --- a/frontend/src/app/common/type/datasetVersionFileTree.spec.ts +++ b/frontend/src/app/common/type/datasetVersionFileTree.spec.ts @@ -41,7 +41,7 @@ describe("getRelativePathFromDatasetFileNode", () => { const node: DatasetFileNode = { name: "tw1.csv", type: "file", - parentDir: "/datasets/bob@texera.com/twitterDataset/v1/california/irvine", + parentDir: "/dataset/bob@texera.com/twitterDataset/v1/california/irvine", }; expect(getRelativePathFromDatasetFileNode(node)).toBe("california/irvine/tw1.csv"); }); @@ -50,7 +50,7 @@ describe("getRelativePathFromDatasetFileNode", () => { const node: DatasetFileNode = { name: "readme.txt", type: "file", - parentDir: "/datasets/bob@texera.com/twitterDataset/v1", + parentDir: "/dataset/bob@texera.com/twitterDataset/v1", }; expect(getRelativePathFromDatasetFileNode(node)).toBe("readme.txt"); }); @@ -59,7 +59,7 @@ describe("getRelativePathFromDatasetFileNode", () => { const node: DatasetFileNode = { name: "v1", type: "directory", - parentDir: "/datasets/bob@texera.com/twitterDataset", + parentDir: "/dataset/bob@texera.com/twitterDataset", }; expect(getRelativePathFromDatasetFileNode(node)).toBe(""); }); @@ -68,7 +68,7 @@ describe("getRelativePathFromDatasetFileNode", () => { const node: DatasetFileNode = { name: "f.csv", type: "file", - parentDir: "/datasets/bob@texera.com/twitterDataset//v1/sub", + parentDir: "/dataset/bob@texera.com/twitterDataset//v1/sub", }; expect(getRelativePathFromDatasetFileNode(node)).toBe("sub/f.csv"); }); diff --git a/frontend/src/app/common/type/datasetVersionFileTree.ts b/frontend/src/app/common/type/datasetVersionFileTree.ts index 98f898d4432..efbb875128c 100644 --- a/frontend/src/app/common/type/datasetVersionFileTree.ts +++ b/frontend/src/app/common/type/datasetVersionFileTree.ts @@ -32,7 +32,7 @@ export function getFullPathFromDatasetFileNode(node: DatasetFileNode): string { /** * Returns the relative path of a DatasetFileNode by stripping the first four segments - * (datasets/ownerEmail/datasetName/versionName). + * (dataset/ownerEmail/datasetName/versionName). * @param node The DatasetFileNode whose relative path is needed. * @returns The relative path (without the first four segments and without a leading slash). */ diff --git a/frontend/src/app/common/type/resource-type.ts b/frontend/src/app/common/type/resource-type.ts index a129c917e27..22c4a8592a9 100644 --- a/frontend/src/app/common/type/resource-type.ts +++ b/frontend/src/app/common/type/resource-type.ts @@ -17,8 +17,8 @@ */ /** - * The leading segment of a logical file path, identifying the resource kind (e.g. /datasets/...). + * The leading segment of a logical file path, identifying the resource kind (e.g. /dataset/...). */ export enum ResourceType { - Datasets = "datasets", + Dataset = "dataset", } diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.spec.ts b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.spec.ts index 78c35761a46..7bb6af6e0db 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.spec.ts @@ -2694,7 +2694,7 @@ describe("DatasetDetailComponent rendered template", () => { const leaf = (name: string): DatasetFileNode => ({ name, type: "file", - parentDir: `/datasets/${OWNER}/ds/v1/nested`, + parentDir: `/dataset/${OWNER}/ds/v1/nested`, size: 2048, }); @@ -2719,7 +2719,7 @@ describe("DatasetDetailComponent rendered template", () => { // The heading is the full path — the copy-path button beside it copies // exactly this string — not the bare file name or the relative path. expect(text(q(fixture.nativeElement, ".file-title-main"))).toBe( - `/datasets/${OWNER}/ds/v1/nested/b.csv` + `/dataset/${OWNER}/ds/v1/nested/b.csv` ); // 2048 bytes reaches the reader as a human-readable size, not as a raw count. expect(text(q(fixture.nativeElement, ".file-size"))).toBe("2.00 KB"); diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-version-filetree/user-dataset-version-filetree.component.spec.ts b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-version-filetree/user-dataset-version-filetree.component.spec.ts index 806d2a6f257..ff9e82609a3 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-version-filetree/user-dataset-version-filetree.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-version-filetree/user-dataset-version-filetree.component.spec.ts @@ -183,7 +183,7 @@ describe("UserDatasetVersionFiletreeComponent", () => { const file = (name: string): DatasetFileNode => ({ name, type: "file", - parentDir: "/datasets/owner/dataset/v1", + parentDir: "/dataset/owner/dataset/v1", }); // Deliberately synchronous: the tree keeps a pending timer, so awaiting whenStable() @@ -262,9 +262,9 @@ describe("UserDatasetVersionFiletreeComponent", () => { const emitted: string[] = []; component.setCoverImage.subscribe((path: string) => emitted.push(path)); - // parentDir has exactly the four stripped segments (datasets/owner/dataset/version), + // parentDir has exactly the four stripped segments (dataset/owner/dataset/version), // so the relative path is just the file name. - component.onSetCover({ name: "photo.png", type: "file", parentDir: "/datasets/owner/dataset/v1" }); + component.onSetCover({ name: "photo.png", type: "file", parentDir: "/dataset/owner/dataset/v1" }); expect(emitted).toEqual(["photo.png"]); }); diff --git a/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.spec.ts b/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.spec.ts index 1eb0d1e6463..b9d70be209f 100644 --- a/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.spec.ts +++ b/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.spec.ts @@ -108,7 +108,7 @@ describe("DatasetSelectionModalComponent", () => { it("ngOnInit initializes selectedDataset and selectedVersion from data.selectedPath", () => { modalData.fileMode = true; - modalData.selectedPath = `/datasets/${OWNER}/myds/v1`; + modalData.selectedPath = `/dataset/${OWNER}/myds/v1`; build(); @@ -151,7 +151,7 @@ describe("DatasetSelectionModalComponent", () => { expect(datasetService.retrieveDatasetVersionFileTree).toHaveBeenCalledWith(10, 100); expect(component.fileTree).toEqual([fileNode]); - expect(component.selectedPath).toBe(`/datasets/${OWNER}/myds/v1`); + expect(component.selectedPath).toBe(`/dataset/${OWNER}/myds/v1`); }); it("onFileSelected sets selectedPath to the node's full path in file mode", () => { diff --git a/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.ts b/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.ts index 97ecf579ca9..b5768aa685b 100644 --- a/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.ts +++ b/frontend/src/app/workspace/component/dataset-selection-modal/dataset-selection-modal.component.ts @@ -85,7 +85,7 @@ export class DatasetSelectionModalComponent implements OnInit { const selectedPath = this.data.selectedPath; if (selectedPath) { const segments = selectedPath.split("/").filter(part => part.length > 0); - // TODO(datasets-prefix): require the prefix once all ml model support PRs are done. + // TODO(dataset-prefix): require the prefix once all ml model support PRs are done. if ((Object.values(ResourceType) as string[]).includes(segments[0])) { segments.shift(); } @@ -124,7 +124,7 @@ export class DatasetSelectionModalComponent implements OnInit { this.fileTree = data.fileNodes; }); if (!this.data.fileMode) { - this.selectedPath = `/${ResourceType.Datasets}/${this.selectedDataset.ownerEmail}/${this.selectedDataset.dataset.name}/${this.selectedVersion.name}`; + this.selectedPath = `/${ResourceType.Dataset}/${this.selectedDataset.ownerEmail}/${this.selectedDataset.dataset.name}/${this.selectedVersion.name}`; } } } diff --git a/sql/changelog.xml b/sql/changelog.xml index d3a7d55228c..9eaa5733a03 100644 --- a/sql/changelog.xml +++ b/sql/changelog.xml @@ -89,7 +89,7 @@ - + diff --git a/sql/updates/36.sql b/sql/updates/36.sql index 5e3c1a182e5..1575c274526 100644 --- a/sql/updates/36.sql +++ b/sql/updates/36.sql @@ -24,19 +24,27 @@ SET search_path TO texera_db; BEGIN; -- The file resolver now requires an explicit resource-type prefix on dataset --- logical paths (/datasets/ownerEmail/datasetName/versionName/...) so other --- resource types (e.g. models) can be told apart by the prefix. Existing --- workflows store unprefixed dataset paths inside workflow.content and --- workflow_version.content, in two operator properties: +-- logical paths (/dataset/ownerEmail/datasetName/versionName/...) so other +-- resource types (e.g. models) can be told apart by the prefix. Stored +-- workflows carry such paths inside workflow.content and workflow_version.content, +-- in two operator properties: -- * fileName (scan-source operators): /owner/name/version/file -- * datasetVersionPath (file-lister operator): /owner/name/version --- This migration prepends the "datasets" segment to both. -- --- A value is treated as a dataset path only when its first two segments match an --- existing (user.email, dataset.name) pair -- that pair is unique. --- Local file paths and URLs match no dataset and are left untouched. --- Already-prefixed values are skipped (idempotent). jsonb_set --- uses create_missing = false so absent properties are never added. +-- This migration brings both to the current form, in two cases: +-- 1. No prefix at all (the original legacy shape) -> prepend "dataset". +-- A value is treated as a dataset path only when its first two segments match +-- an existing (user.email, dataset.name) pair -- that pair is unique. Local +-- file paths and URLs match no dataset and are left untouched. +-- 2. A plural prefix, written by an earlier revision of this migration or by a +-- build from before the prefix was singularized -> rewrite the leading segment +-- ("datasets" -> "dataset", "models" -> "model"). Case 2 is tested first, since +-- such a path has no prefix by case 1's reckoning yet must not be prefixed again. +-- +-- Only the leading segment is ever rewritten, so a local path that merely contains +-- "/datasets/" further along keeps it. Values already in the current form match +-- neither case, which makes the migration idempotent. jsonb_set uses +-- create_missing = false so absent properties are never added. DO $$ DECLARE @@ -59,12 +67,16 @@ BEGIN ) f WHERE jsonb_typeof(w.content::jsonb -> 'operators') = 'array' AND ( - (f.fn IS NOT NULL AND left(f.fn, 10) <> '/datasets/' + f.fn ~ '^/(dataset|model)s/' + OR + f.dvp ~ '^/(dataset|model)s/' + OR + (f.fn IS NOT NULL AND left(f.fn, 9) <> '/dataset/' AND EXISTS (SELECT 1 FROM dataset d JOIN "user" u ON d.owner_uid = u.uid WHERE u.email = split_part(ltrim(f.fn, '/'), '/', 1) AND d.name = split_part(ltrim(f.fn, '/'), '/', 2))) OR - (f.dvp IS NOT NULL AND left(f.dvp, 10) <> '/datasets/' + (f.dvp IS NOT NULL AND left(f.dvp, 9) <> '/dataset/' AND EXISTS (SELECT 1 FROM dataset d JOIN "user" u ON d.owner_uid = u.uid WHERE u.email = split_part(ltrim(f.dvp, '/'), '/', 1) AND d.name = split_part(ltrim(f.dvp, '/'), '/', 2))) @@ -80,22 +92,26 @@ BEGIN op, '{operatorProperties,fileName}', CASE - WHEN f.fn IS NOT NULL AND left(f.fn, 10) <> '/datasets/' + WHEN f.fn ~ '^/(dataset|model)s/' + THEN to_jsonb(regexp_replace(f.fn, '^/(dataset|model)s/', '/\1/')) + WHEN f.fn IS NOT NULL AND left(f.fn, 9) <> '/dataset/' AND EXISTS (SELECT 1 FROM dataset d JOIN "user" u ON d.owner_uid = u.uid WHERE u.email = split_part(ltrim(f.fn, '/'), '/', 1) AND d.name = split_part(ltrim(f.fn, '/'), '/', 2)) - THEN to_jsonb('/datasets/' || ltrim(f.fn, '/')) + THEN to_jsonb('/dataset/' || ltrim(f.fn, '/')) ELSE COALESCE(op #> '{operatorProperties,fileName}', 'null'::jsonb) END, false ), '{operatorProperties,datasetVersionPath}', CASE - WHEN f.dvp IS NOT NULL AND left(f.dvp, 10) <> '/datasets/' + WHEN f.dvp ~ '^/(dataset|model)s/' + THEN to_jsonb(regexp_replace(f.dvp, '^/(dataset|model)s/', '/\1/')) + WHEN f.dvp IS NOT NULL AND left(f.dvp, 9) <> '/dataset/' AND EXISTS (SELECT 1 FROM dataset d JOIN "user" u ON d.owner_uid = u.uid WHERE u.email = split_part(ltrim(f.dvp, '/'), '/', 1) AND d.name = split_part(ltrim(f.dvp, '/'), '/', 2)) - THEN to_jsonb('/datasets/' || ltrim(f.dvp, '/')) + THEN to_jsonb('/dataset/' || ltrim(f.dvp, '/')) ELSE COALESCE(op #> '{operatorProperties,datasetVersionPath}', 'null'::jsonb) END, false @@ -136,12 +152,16 @@ BEGIN ) f WHERE jsonb_typeof(wv.content::jsonb -> 'operators') = 'array' AND ( - (f.fn IS NOT NULL AND left(f.fn, 10) <> '/datasets/' + f.fn ~ '^/(dataset|model)s/' + OR + f.dvp ~ '^/(dataset|model)s/' + OR + (f.fn IS NOT NULL AND left(f.fn, 9) <> '/dataset/' AND EXISTS (SELECT 1 FROM dataset d JOIN "user" u ON d.owner_uid = u.uid WHERE u.email = split_part(ltrim(f.fn, '/'), '/', 1) AND d.name = split_part(ltrim(f.fn, '/'), '/', 2))) OR - (f.dvp IS NOT NULL AND left(f.dvp, 10) <> '/datasets/' + (f.dvp IS NOT NULL AND left(f.dvp, 9) <> '/dataset/' AND EXISTS (SELECT 1 FROM dataset d JOIN "user" u ON d.owner_uid = u.uid WHERE u.email = split_part(ltrim(f.dvp, '/'), '/', 1) AND d.name = split_part(ltrim(f.dvp, '/'), '/', 2))) @@ -157,22 +177,26 @@ BEGIN op, '{operatorProperties,fileName}', CASE - WHEN f.fn IS NOT NULL AND left(f.fn, 10) <> '/datasets/' + WHEN f.fn ~ '^/(dataset|model)s/' + THEN to_jsonb(regexp_replace(f.fn, '^/(dataset|model)s/', '/\1/')) + WHEN f.fn IS NOT NULL AND left(f.fn, 9) <> '/dataset/' AND EXISTS (SELECT 1 FROM dataset d JOIN "user" u ON d.owner_uid = u.uid WHERE u.email = split_part(ltrim(f.fn, '/'), '/', 1) AND d.name = split_part(ltrim(f.fn, '/'), '/', 2)) - THEN to_jsonb('/datasets/' || ltrim(f.fn, '/')) + THEN to_jsonb('/dataset/' || ltrim(f.fn, '/')) ELSE COALESCE(op #> '{operatorProperties,fileName}', 'null'::jsonb) END, false ), '{operatorProperties,datasetVersionPath}', CASE - WHEN f.dvp IS NOT NULL AND left(f.dvp, 10) <> '/datasets/' + WHEN f.dvp ~ '^/(dataset|model)s/' + THEN to_jsonb(regexp_replace(f.dvp, '^/(dataset|model)s/', '/\1/')) + WHEN f.dvp IS NOT NULL AND left(f.dvp, 9) <> '/dataset/' AND EXISTS (SELECT 1 FROM dataset d JOIN "user" u ON d.owner_uid = u.uid WHERE u.email = split_part(ltrim(f.dvp, '/'), '/', 1) AND d.name = split_part(ltrim(f.dvp, '/'), '/', 2)) - THEN to_jsonb('/datasets/' || ltrim(f.dvp, '/')) + THEN to_jsonb('/dataset/' || ltrim(f.dvp, '/')) ELSE COALESCE(op #> '{operatorProperties,datasetVersionPath}', 'null'::jsonb) END, false @@ -197,7 +221,7 @@ BEGIN ) SELECT count(*) INTO wv_count FROM updated; - RAISE NOTICE 'Prefixed legacy dataset paths with "datasets/" in % workflow and % workflow_version row(s).', wf_count, wv_count; + RAISE NOTICE 'Normalized the resource-type path prefix in % workflow and % workflow_version row(s).', wf_count, wv_count; END $$; COMMIT;