From 638a7934e9d89620c1e7b89307851ab6f556d22e Mon Sep 17 00:00:00 2001 From: Rafal Hawrylak Date: Wed, 19 Aug 2026 13:58:58 +0000 Subject: [PATCH] Support ref() and DAG dependencies in PropertyGraph Extends PropertyGraph to accept `ref: [dataset, table]` in place of a bare table name for `nodeTables` and `edgeTables`. Refs resolve through the same `Session.resolve` path used by tables and operations so the graph action inherits the standard dependency edges. The resolved dependencies land on the compiled action's `dependencyTargets` which makes the executor run source tables before the graph DDL. Ref resolution respects schema suffixes and database overrides so a graph declared with `ref: [authors]` in a suffixed workspace still targets the suffixed dataset. Lock-in tests cover the resolved-DDL output for named refs, dataset-scoped refs, mixed literal-and-ref entries, and the error surface when a ref points at a nonexistent action. --- core/actions/property_graph.ts | 110 +++- core/actions/property_graph_test.ts | 143 ++++- core/main_test.ts | 881 ++++++++++++++++++++++++++++ core/session.ts | 10 + core/utils.ts | 4 +- 5 files changed, 1141 insertions(+), 7 deletions(-) diff --git a/core/actions/property_graph.ts b/core/actions/property_graph.ts index b7b77a5ba..fd31fe54f 100644 --- a/core/actions/property_graph.ts +++ b/core/actions/property_graph.ts @@ -1,15 +1,32 @@ import { verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "df/common/protos"; import { ActionBuilder } from "df/core/actions"; +import { Declaration } from "df/core/actions/declaration"; import { Session } from "df/core/session"; +import { checkAssertionsForDependency } from "df/core/utils"; import { dataform } from "df/protos/ts"; const CATALOG_NOT_SUPPORTED_MESSAGE = "Catalog-based data sources (BigLake/Iceberg/external catalogs) are not yet supported."; +function entityRefKey(name: string): string { + return `entity:${name}`; +} + +function relationshipRefKey(name: string): string { + return `relationship:${name}`; +} + export class PropertyGraph extends ActionBuilder { public session: Session; + public dependOnDependencyAssertions: boolean = false; private proto = dataform.PropertyGraph.create({ description: "", disabled: false }); + // Populated during construction for entity/relationship configs that use a ref-style + // dataSource; drained in finalize() once all actions are indexed, since the target + // schema/database of a ref cannot be resolved until every action is known. + // Keys are produced by entityRefKey / relationshipRefKey. + private pendingRefs = new Map(); + private dependencyKeys = new Set(); constructor(session?: Session, unverifiedConfig?: any, filename?: string) { super(session); @@ -61,8 +78,6 @@ export class PropertyGraph extends ActionBuilder { this.validateUniqueNames(config.name); this.resolveEndpointDefaults(); this.validateEndpointReferences(config.name); - - this.proto.graphBody = this.emitGraphBody(); } public getFileName() { @@ -74,20 +89,66 @@ export class PropertyGraph extends ActionBuilder { } public compile() { - return verifyObjectMatchesProto( + this.proto = verifyObjectMatchesProto( dataform.PropertyGraph, this.proto, VerifyProtoErrorBehaviour.SUGGEST_REPORTING_TO_DATAFORM_TEAM ); + return this.proto; + } + + public finalize() { + let allResolved = true; + for (const entity of this.proto.entities) { + if (!this.resolveRefIntoDataSource(entity, entityRefKey(entity.name))) { + allResolved = false; + } + } + for (const rel of this.proto.relationships) { + if (!this.resolveRefIntoDataSource(rel, relationshipRefKey(rel.name))) { + allResolved = false; + } + } + if (allResolved) { + this.proto.graphBody = this.emitGraphBody(); + } + } + + private resolveRefIntoDataSource( + entityOrRel: dataform.IGraphEntity | dataform.IGraphRelationship, + key: string + ): boolean { + const rawRef = this.pendingRefs.get(key); + if (!rawRef) { + return true; + } + const resolved = this.session.indexedActions.find(rawRef); + if (resolved.length !== 1) { + return false; + } + const resolvedAction = resolved[0]; + const target = resolvedAction.getTarget(); + if (resolvedAction instanceof Declaration) { + entityOrRel.dataSource = target; + } else { + entityOrRel.dataSource = dataform.Target.create({ + database: target.database && this.session.finalizeDatabase(target.database), + schema: this.session.finalizeSchema(target.schema), + name: this.session.finalizeName(target.name) + }); + } + return true; } private normalizeEntitiesAndRelationships(unverifiedConfig: any) { for (const entity of unverifiedConfig.entities || []) { + normalizeRef(entity); normalizeKeys(entity); normalizeFields(entity); normalizeFieldsOnLabels(entity); } for (const relationship of unverifiedConfig.relationships || []) { + normalizeRef(relationship); normalizeKeys(relationship); normalizeFields(relationship); normalizeFieldsOnLabels(relationship); @@ -113,7 +174,7 @@ export class PropertyGraph extends ActionBuilder { } const entityName = entityConfig.name; const where = `Property graph '${graphName}': entity '${entityName}'`; - const dataSource = this.resolveDataSource(entityConfig, where); + const dataSource = this.resolveDataSource(entityConfig, entityRefKey(entityName), where); if (!entityConfig.keys || entityConfig.keys.length === 0) { throw new Error(`${where} must declare 'keys'.`); } @@ -137,7 +198,7 @@ export class PropertyGraph extends ActionBuilder { } const relName = relConfig.name; const where = `Property graph '${graphName}': relationship '${relName}'`; - const dataSource = this.resolveDataSource(relConfig, where); + const dataSource = this.resolveDataSource(relConfig, relationshipRefKey(relName), where); if (!relConfig.source) { throw new Error(`${where} must declare 'source'.`); @@ -163,6 +224,7 @@ export class PropertyGraph extends ActionBuilder { private resolveDataSource( entityOrRel: dataform.IGraphEntityConfig | dataform.IGraphRelationshipConfig, + pendingKey: string, where: string ): dataform.Target { if (entityOrRel.dataSourceCatalog) { @@ -187,6 +249,29 @@ export class PropertyGraph extends ActionBuilder { } return dataform.Target.create({ name: ds.table, schema: dataset, database: project }); } + if (entityOrRel.dataSourceRef) { + const ref = entityOrRel.dataSourceRef; + if (!ref.name) { + throw new Error(`${where}: 'ref' must include a 'name'.`); + } + const rawRef: dataform.ITarget = { name: ref.name }; + if (ref.schema) { + rawRef.schema = ref.schema; + } + if (ref.database) { + rawRef.database = ref.database; + } + this.pendingRefs.set(pendingKey, rawRef); + const depKey = `${rawRef.database || ""}.${rawRef.schema || ""}.${rawRef.name}`; + if (!this.dependencyKeys.has(depKey)) { + this.dependencyKeys.add(depKey); + const depTarget = checkAssertionsForDependency(this, rawRef); + if (depTarget) { + this.proto.dependencyTargets.push(depTarget); + } + } + return undefined; + } throw new Error(`${where}: must declare a data source.`); } @@ -268,6 +353,21 @@ export class PropertyGraph extends ActionBuilder { } } +function normalizeRef(entityOrRel: any) { + if (entityOrRel.ref === undefined || entityOrRel.ref === null) { + return; + } + if (typeof entityOrRel.ref === "string") { + entityOrRel.dataSourceRef = { name: entityOrRel.ref }; + delete entityOrRel.ref; + return; + } + if (typeof entityOrRel.ref === "object") { + entityOrRel.dataSourceRef = entityOrRel.ref; + delete entityOrRel.ref; + } +} + function normalizeKeys(entityOrRel: any) { if (typeof entityOrRel.keys === "string") { entityOrRel.keys = [entityOrRel.keys]; diff --git a/core/actions/property_graph_test.ts b/core/actions/property_graph_test.ts index 4b292caa3..8fe7e1f79 100644 --- a/core/actions/property_graph_test.ts +++ b/core/actions/property_graph_test.ts @@ -16,7 +16,14 @@ function makeSession(): Session { } function compile(config: any, filename = "definitions/graph.yaml"): dataform.PropertyGraph { - return new PropertyGraph(makeSession(), config, filename).compile(); + const action = new PropertyGraph(makeSession(), config, filename); + const compiled = action.compile(); + action.finalize(); + return compiled; +} + +function build(config: any, filename = "definitions/graph.yaml"): PropertyGraph { + return new PropertyGraph(makeSession(), config, filename); } const graphTarget = (name: string) => ({ @@ -1352,4 +1359,138 @@ suite("property_graph", () => { }) ); }); + + test("scalar ref normalizes to dataSourceRef and populates dependencyTargets", () => { + const compiled = build({ + name: "G", + entities: [ + { + name: "A", + ref: "books", + keys: ["id"] + } + ] + }).compile(); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + dependencyTargets: [{ name: "books", includeDependentAssertions: false }], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [{ name: "A", keys: ["id"] }] + }) + ); + }); + + test("object ref preserves schema and database in dependencyTargets", () => { + const compiled = build({ + name: "G", + entities: [ + { + name: "A", + ref: { name: "books", schema: "analytics", database: "proj" }, + keys: ["id"] + } + ] + }).compile(); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + dependencyTargets: [ + { database: "proj", schema: "analytics", name: "books", includeDependentAssertions: false } + ], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [{ name: "A", keys: ["id"] }] + }) + ); + }); + + test("duplicate refs across entities dedupe in dependencyTargets", () => { + const compiled = build({ + name: "G", + entities: [ + { name: "A", ref: "books", keys: ["id"] }, + { name: "B", ref: "books", keys: ["id"] } + ] + }).compile(); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + dependencyTargets: [{ name: "books", includeDependentAssertions: false }], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { name: "A", keys: ["id"] }, + { name: "B", keys: ["id"] } + ] + }) + ); + }); + + test("ref on relationship also normalizes and populates dependencyTargets", () => { + const compiled = build({ + name: "G", + entities: [ + { name: "A", dataSourceString: "p.d.A", keys: ["id"] }, + { name: "B", dataSourceString: "p.d.B", keys: ["id"] } + ], + relationships: [ + { + name: "R", + ref: "wrote", + keys: ["a_id", "b_id"], + source: { entity: "A", joinKeys: ["a_id"] }, + destination: { entity: "B", joinKeys: ["b_id"] } + } + ] + }).compile(); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + dependencyTargets: [{ name: "wrote", includeDependentAssertions: false }], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { name: "A", dataSource: { database: "p", schema: "d", name: "A" }, keys: ["id"] }, + { name: "B", dataSource: { database: "p", schema: "d", name: "B" }, keys: ["id"] } + ], + relationships: [ + { + name: "R", + keys: ["a_id", "b_id"], + source: { entity: "A", relationshipColumns: ["a_id"], entityColumns: ["id"] }, + destination: { entity: "B", relationshipColumns: ["b_id"], entityColumns: ["id"] } + } + ] + }) + ); + }); + + test("errors when ref has no name", () => { + expect(() => + compile({ + name: "G", + entities: [ + { + name: "A", + ref: { schema: "s" }, + keys: ["id"] + } + ] + }) + ).to.throw("'ref' must include a 'name'"); + }); }); diff --git a/core/main_test.ts b/core/main_test.ts index d853ceac6..2fc498cdc 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -2890,6 +2890,802 @@ relationships: })); }); + test("ref to declaration resolves entity dataSource and renders graphBody", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/actions.yaml"), + ` +actions: +- declaration: + name: books +` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: RefGraph +entities: +- name: Book + ref: books + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.propertyGraphs)).deep.equals( + asPlainObject([ + { + target: { + schema: "defaultDataset", + name: "RefGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "RefGraph", + database: "defaultProject" + }, + dependencyTargets: [ + { + database: "defaultProject", + schema: "defaultDataset", + name: "books" + } + ], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { + name: "Book", + dataSource: { + schema: "defaultDataset", + name: "books", + database: "defaultProject" + }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.defaultDataset.books` AS Book KEY (id)\n" + + ")" + } + ]) + ); + }); + + test("ref with schema override resolves the matching declaration", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/actions.yaml"), + ` +actions: +- declaration: + name: books + dataset: alt +- declaration: + name: books +` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: RefWithSchemaGraph +entities: +- name: Book + ref: + name: books + schema: alt + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.propertyGraphs)).deep.equals( + asPlainObject([ + { + target: { + schema: "defaultDataset", + name: "RefWithSchemaGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "RefWithSchemaGraph", + database: "defaultProject" + }, + dependencyTargets: [ + { + database: "defaultProject", + schema: "alt", + name: "books" + } + ], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { + name: "Book", + dataSource: { + schema: "alt", + name: "books", + database: "defaultProject" + }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.alt.books` AS Book KEY (id)\n" + + ")" + } + ]) + ); + }); + + test("missing ref emits a compilation error and leaves graphBody empty", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: MissingRefGraph +entities: +- name: Book + ref: nonexistent + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + const missingRefTarget = { + schema: "defaultDataset", + name: "MissingRefGraph", + database: "defaultProject" + }; + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: { + compilationErrors: [ + graphError( + "definitions/graph.yaml", + "Missing dependency detected: Action " + + "\"defaultProject.defaultDataset.MissingRefGraph\" depends on " + + "\"{\"name\":\"nonexistent\",\"includeDependentAssertions\":false}\" " + + "which does not exist", + { + actionName: "defaultProject.defaultDataset.MissingRefGraph", + actionTarget: missingRefTarget + } + ) + ] + }, + dataformCoreVersion: version, + targets: [missingRefTarget], + jitData: {}, + propertyGraphs: [ + { + target: missingRefTarget, + canonicalTarget: missingRefTarget, + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { + name: "Book", + keys: ["id"] + } + ] + } + ] + })); + }); + + test("ref to a table respects datasetSuffix on the resolved dependency", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + ` +defaultProject: defaultProject +defaultDataset: defaultDataset +defaultLocation: US +datasetSuffix: dev +` + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/books.sqlx"), + `config {type: "table"} +select 1 as id` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: SuffixRefGraph +entities: +- name: Book + ref: books + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.propertyGraphs)).deep.equals( + asPlainObject([ + { + target: { + schema: "defaultDataset_dev", + name: "SuffixRefGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "SuffixRefGraph", + database: "defaultProject" + }, + dependencyTargets: [ + { + schema: "defaultDataset_dev", + name: "books", + database: "defaultProject" + } + ], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { + name: "Book", + dataSource: { + schema: "defaultDataset_dev", + name: "books", + database: "defaultProject" + }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.defaultDataset_dev.books` AS Book KEY (id)\n" + + ")" + } + ]) + ); + }); + + test("ref with database override resolves the matching declaration", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/actions.yaml"), + ` +actions: +- declaration: + name: books + project: otherProject +- declaration: + name: books +` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: RefWithDatabaseGraph +entities: +- name: Book + ref: + name: books + database: otherProject + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.propertyGraphs)).deep.equals( + asPlainObject([ + { + target: { + schema: "defaultDataset", + name: "RefWithDatabaseGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "RefWithDatabaseGraph", + database: "defaultProject" + }, + dependencyTargets: [ + { + database: "otherProject", + schema: "defaultDataset", + name: "books" + } + ], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { + name: "Book", + dataSource: { + database: "otherProject", + schema: "defaultDataset", + name: "books" + }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n" + + " `otherProject.defaultDataset.books` AS Book KEY (id)\n" + + ")" + } + ]) + ); + }); + + test("relationship ref resolves through the full pipeline", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/actions.yaml"), + ` +actions: +- declaration: + name: wrote +` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: RelationshipRefGraph +entities: +- name: Book + dataSourceString: defaultProject.defaultDataset.books + keys: + - id +- name: Author + dataSourceString: defaultProject.defaultDataset.authors + keys: + - id +relationships: +- name: WrittenBy + ref: wrote + keys: + - author_id + - book_id + source: + entity: Book + joinKeys: + - book_id + destination: + entity: Author + joinKeys: + - author_id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.propertyGraphs)).deep.equals( + asPlainObject([ + { + target: { + schema: "defaultDataset", + name: "RelationshipRefGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "RelationshipRefGraph", + database: "defaultProject" + }, + dependencyTargets: [ + { + database: "defaultProject", + schema: "defaultDataset", + name: "wrote" + } + ], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { + name: "Book", + dataSource: { + database: "defaultProject", + schema: "defaultDataset", + name: "books" + }, + keys: ["id"] + }, + { + name: "Author", + dataSource: { + database: "defaultProject", + schema: "defaultDataset", + name: "authors" + }, + keys: ["id"] + } + ], + relationships: [ + { + name: "WrittenBy", + dataSource: { + database: "defaultProject", + schema: "defaultDataset", + name: "wrote" + }, + keys: ["author_id", "book_id"], + source: { + entity: "Book", + relationshipColumns: ["book_id"], + entityColumns: ["id"] + }, + destination: { + entity: "Author", + relationshipColumns: ["author_id"], + entityColumns: ["id"] + } + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.defaultDataset.books` AS Book KEY (id),\n" + + " `defaultProject.defaultDataset.authors` AS Author KEY (id)\n" + + ")\n" + + "EDGE TABLES (\n" + + " `defaultProject.defaultDataset.wrote` AS WrittenBy " + + "KEY (author_id, book_id) " + + "SOURCE KEY (book_id) REFERENCES Book (id) " + + "DESTINATION KEY (author_id) REFERENCES Author (id)\n" + + ")" + } + ]) + ); + }); + + test("ref to a view resolves and picks up datasetSuffix", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + ` +defaultProject: defaultProject +defaultDataset: defaultDataset +defaultLocation: US +datasetSuffix: dev +` + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/books.sqlx"), + `config {type: "view"} +select 1 as id` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: ViewRefGraph +entities: +- name: Book + ref: books + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.propertyGraphs)).deep.equals( + asPlainObject([ + { + target: { + schema: "defaultDataset_dev", + name: "ViewRefGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "ViewRefGraph", + database: "defaultProject" + }, + dependencyTargets: [ + { + database: "defaultProject", + schema: "defaultDataset_dev", + name: "books" + } + ], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { + name: "Book", + dataSource: { + database: "defaultProject", + schema: "defaultDataset_dev", + name: "books" + }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.defaultDataset_dev.books` AS Book KEY (id)\n" + + ")" + } + ]) + ); + }); + + test("ambiguous ref emits a compilation error", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/actions.yaml"), + ` +actions: +- declaration: + name: books + dataset: one +- declaration: + name: books + dataset: two +` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: AmbiguousRefGraph +entities: +- name: Book + ref: books + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + const declOneTarget = { schema: "one", name: "books", database: "defaultProject" }; + const declTwoTarget = { schema: "two", name: "books", database: "defaultProject" }; + const graphTarget = { + schema: "defaultDataset", + name: "AmbiguousRefGraph", + database: "defaultProject" + }; + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: { + compilationErrors: [ + graphError( + "definitions/graph.yaml", + `Ambiguous Action name: {"name":"books","includeDependentAssertions":false}. ` + + "Did you mean one of: one.books, two.books.", + { + actionName: "defaultProject.defaultDataset.AmbiguousRefGraph", + actionTarget: graphTarget + } + ) + ] + }, + dataformCoreVersion: version, + targets: [declOneTarget, declTwoTarget, graphTarget], + jitData: {}, + declarations: [ + { target: declOneTarget, canonicalTarget: declOneTarget }, + { target: declTwoTarget, canonicalTarget: declTwoTarget } + ], + propertyGraphs: [ + { + target: graphTarget, + canonicalTarget: graphTarget, + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [{ name: "Book", keys: ["id"] }] + } + ] + })); + }); + + test("ref to a table respects projectSuffix on the resolved dependency", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + ` +defaultProject: defaultProject +defaultDataset: defaultDataset +defaultLocation: US +projectSuffix: dev +` + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/books.sqlx"), + `config {type: "table"} +select 1 as id` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: ProjectSuffixRefGraph +entities: +- name: Book + ref: books + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.propertyGraphs)).deep.equals( + asPlainObject([ + { + target: { + schema: "defaultDataset", + name: "ProjectSuffixRefGraph", + database: "defaultProject_dev" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "ProjectSuffixRefGraph", + database: "defaultProject" + }, + dependencyTargets: [ + { + schema: "defaultDataset", + name: "books", + database: "defaultProject_dev" + } + ], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { + name: "Book", + dataSource: { + schema: "defaultDataset", + name: "books", + database: "defaultProject_dev" + }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject_dev.defaultDataset.books` AS Book KEY (id)\n" + + ")" + } + ]) + ); + }); + + test("ref to a table respects namePrefix on the resolved dependency", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + ` +defaultProject: defaultProject +defaultDataset: defaultDataset +defaultLocation: US +namePrefix: pfx +` + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/books.sqlx"), + `config {type: "table"} +select 1 as id` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: NamePrefixRefGraph +entities: +- name: Book + ref: books + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.propertyGraphs)).deep.equals( + asPlainObject([ + { + target: { + schema: "defaultDataset", + name: "pfx_NamePrefixRefGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "NamePrefixRefGraph", + database: "defaultProject" + }, + dependencyTargets: [ + { + schema: "defaultDataset", + name: "pfx_books", + database: "defaultProject" + } + ], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { + name: "Book", + dataSource: { + schema: "defaultDataset", + name: "pfx_books", + database: "defaultProject" + }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.defaultDataset.pfx_books` AS Book KEY (id)\n" + + ")" + } + ]) + ); + }); + test("graph target colliding with a table target is flagged as duplicate", () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( @@ -3078,5 +3874,90 @@ relationships: ] })); }); + + test("mixed ref and dataSourceString: only ref target appears in dependencyTargets", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/books.sqlx"), + `config {type: "table"} +select 1 as id` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/authors.sqlx"), + `config {type: "table"} +select 1 as id` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: MixedRefStringGraph +entities: +- name: Book + ref: books + keys: + - id +- name: Author + dataSourceString: defaultProject.defaultDataset.authors + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.propertyGraphs)).deep.equals( + asPlainObject([ + { + target: { + schema: "defaultDataset", + name: "MixedRefStringGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "MixedRefStringGraph", + database: "defaultProject" + }, + dependencyTargets: [ + { database: "defaultProject", schema: "defaultDataset", name: "books" } + ], + fileName: "definitions/graph.yaml", + description: "", + disabled: false, + entities: [ + { + name: "Book", + dataSource: { + schema: "defaultDataset", + name: "books", + database: "defaultProject" + }, + keys: ["id"] + }, + { + name: "Author", + dataSource: { + schema: "defaultDataset", + name: "authors", + database: "defaultProject" + }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.defaultDataset.books` AS Book KEY (id),\n" + + " `defaultProject.defaultDataset.authors` AS Author KEY (id)\n" + + ")" + } + ]) + ); + }); }); }); diff --git a/core/session.ts b/core/session.ts index 373a76bfb..fc5e56766 100644 --- a/core/session.ts +++ b/core/session.ts @@ -545,6 +545,8 @@ export class Session { this.removeNonUniqueActionsFromCompiledGraph(compiledGraph); + this.finalizePropertyGraphs(); + this.checkTestNameUniqueness(compiledGraph.tests); this.checkCircularity( @@ -773,6 +775,14 @@ export class Session { }); } + private finalizePropertyGraphs() { + for (const action of this.actions) { + if (action instanceof PropertyGraph) { + action.finalize(); + } + } + } + private removeNonUniqueActionsFromCompiledGraph(compiledGraph: dataform.CompiledGraph) { function getNonUniqueTargets(targets: dataform.ITarget[]): Set { const allTargets = new Set(); diff --git a/core/utils.ts b/core/utils.ts index 960a1e616..b0e19d4b8 100644 --- a/core/utils.ts +++ b/core/utils.ts @@ -4,6 +4,7 @@ import { DataPreparation } from "df/core/actions/data_preparation"; import { IncrementalTable } from "df/core/actions/incremental_table"; import { Notebook } from "df/core/actions/notebook"; import { Operation } from "df/core/actions/operation"; +import { PropertyGraph } from "df/core/actions/property_graph"; import { Table } from "df/core/actions/table"; import { View } from "df/core/actions/view"; import { Contextable, Resolvable } from "df/core/contextables"; @@ -20,7 +21,8 @@ type actionsWithDependencies = | IncrementalTable | Operation | Notebook - | DataPreparation; + | DataPreparation + | PropertyGraph; // This side-steps webpack's require in favour of the real require. export const nativeRequire =