From d34f226b5e8342ef7e97373c73d52ecda3dce0d0 Mon Sep 17 00:00:00 2001 From: Ryan Zhang Date: Wed, 12 Aug 2026 12:26:11 -0700 Subject: [PATCH 01/10] feat(python-notebook-migration, frontend): add AI generate workflow entry point on the dashboard --- .../user-workflow.component.html | 14 ++ .../user-workflow.component.spec.ts | 204 +++++++++++++++- .../user-workflow/user-workflow.component.ts | 130 ++++++++++- .../notebook-import-modal.component.html | 221 +++++++++--------- .../notebook-import-modal.component.scss | 43 +++- .../notebook-import-modal.component.spec.ts | 93 +++++++- .../notebook-import-modal.component.ts | 74 ++++-- .../component/workspace.component.spec.ts | 20 +- .../component/workspace.component.ts | 12 +- .../jupyter-panel/jupyter-panel.service.ts | 12 +- .../notebook-migration.service.spec.ts | 60 ++++- .../notebook-migration.service.ts | 51 +++- 12 files changed, 776 insertions(+), 158 deletions(-) diff --git a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.html b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.html index 492efc53766..d945a21d36e 100644 --- a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.html +++ b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.html @@ -51,6 +51,20 @@

Workflows

nzTheme="outline"> + + - - - Upload Python Jupyter Notebook - - -
- - - + + Selected file: {{ importForm.get('file')?.value?.name }} + +
+
+
- - Selected file: {{ importForm.get('file')?.value?.name }} - - - - + + + Select Model Type + - - - Select Model Type - + + + + + + + + + - - - - - - + - + + + - - - - - - + - diff --git a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.scss b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.scss index 2f898489821..d5d60d1a345 100644 --- a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.scss +++ b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.scss @@ -70,9 +70,46 @@ width: 50%; } - &-warning { - display: block; - margin-bottom: 12px; + // Wraps the form and footer so the loading overlay can cover them without changing the + // modal's height (which would otherwise resize and re-center the centered modal on submit). + &-content { + position: relative; + } + + &-loading { + position: absolute; + inset: 0; + z-index: 1; + // Solid background so the form and footer underneath are fully hidden. + background-color: #fff; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 14px; + padding: 16px 24px 28px; + text-align: center; + } + + &-loading-title { + margin: 8px 0 0; + font-size: 20px; + font-weight: 700; + } + + &-loading-elapsed { + margin: 0; + font-size: 18px; + font-weight: 600; + font-variant-numeric: tabular-nums; + } + + &-loading-text { + margin: 0; + max-width: 480px; + font-size: 16px; + line-height: 1.5; + color: rgba(0, 0, 0, 0.65); } &-footer { diff --git a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.spec.ts b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.spec.ts index eb6d199bd17..6587a89057b 100644 --- a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.spec.ts +++ b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.spec.ts @@ -32,7 +32,7 @@ describe("NotebookImportModalComponent", () => { let fixture: ComponentFixture; let component: NotebookImportModalComponent; let notebookMigrationService: NotebookMigrationService; - let modalRef: { close: ReturnType }; + let modalRef: { close: ReturnType; updateConfig: ReturnType }; // The opener-supplied gate; tests set its resolved value to drive close vs stay-open. let requestImport: ReturnType; @@ -56,19 +56,28 @@ describe("NotebookImportModalComponent", () => { } beforeEach(() => { - modalRef = { close: vi.fn() }; + modalRef = { close: vi.fn(), updateConfig: vi.fn() }; requestImport = vi.fn().mockResolvedValue(true); }); - it("renders the warning, diagram, and a usable model select once models load", async () => { + it("renders the diagram and a usable model select once models load", async () => { await createWith(of([{ name: "gpt-4" }])); const root = fixture.nativeElement as HTMLElement; - expect(root.querySelector(".import-modal-warning")).not.toBeNull(); expect(root.querySelector("img[alt='Notebook to Workflow']")).not.toBeNull(); expect(root.querySelector("nz-select")).not.toBeNull(); expect(root.textContent).toContain("Select a model"); }); + it("shows the loading spinner only while a submission is in flight", async () => { + await createWith(of([{ name: "gpt-4" }])); + const spinning = () => (fixture.nativeElement as HTMLElement).querySelector(".ant-spin-spinning") !== null; + expect(spinning()).toBe(false); + + component.isSubmitting = true; + fixture.detectChanges(); + expect(spinning()).toBe(true); + }); + it("shows the disabled 'no models available' select when the list is empty", async () => { await createWith(of([])); expect((fixture.nativeElement as HTMLElement).textContent).toContain("No models available"); @@ -125,7 +134,7 @@ describe("NotebookImportModalComponent", () => { }); it("onSubmit keeps the modal open when the opener declines", async () => { - // e.g. the user backed out of the overwrite confirmation. + // e.g. generation failed and the user can retry. requestImport.mockResolvedValue(false); await createWith(of([{ name: "gpt-4" }])); component.importForm.setValue({ file: { name: "x.ipynb" } as NzUploadFile, model: "gpt-4" }); @@ -136,6 +145,80 @@ describe("NotebookImportModalComponent", () => { expect(modalRef.close).not.toHaveBeenCalled(); }); + it("locks the modal shut while generating and restores the close controls on failure", async () => { + let resolveRequest!: (proceed: boolean) => void; + requestImport.mockReturnValue(new Promise(resolve => (resolveRequest = resolve))); + await createWith(of([{ name: "gpt-4" }])); + component.importForm.setValue({ file: { name: "x.ipynb" } as NzUploadFile, model: "gpt-4" }); + + const submitting = component.onSubmit(); + // While generation is pending, the X, mask click, and ESC are disabled. + expect(modalRef.updateConfig).toHaveBeenCalledWith({ + nzClosable: false, + nzMaskClosable: false, + nzKeyboard: false, + }); + + resolveRequest(false); // generation failed + await submitting; + // The modal stayed open, so the close controls are restored. + expect(modalRef.updateConfig).toHaveBeenLastCalledWith({ + nzClosable: true, + nzMaskClosable: true, + nzKeyboard: true, + }); + expect(modalRef.close).not.toHaveBeenCalled(); + }); + + it("computes the elapsed time from the start timestamp as mm:ss", async () => { + await createWith(of([{ name: "gpt-4" }])); + // No generation started yet -> no start timestamp -> zero. + expect(component.formattedElapsedTime).toBe("0:00"); + (component as any).startTime = 1000; + vi.spyOn(Date, "now").mockReturnValue(1000 + 62_000); + expect(component.formattedElapsedTime).toBe("1:02"); + }); + + it("runs the stopwatch off wall-clock time while generating and stops the interval when done", async () => { + let resolveRequest!: (proceed: boolean) => void; + requestImport.mockReturnValue(new Promise(resolve => (resolveRequest = resolve))); + await createWith(of([{ name: "gpt-4" }])); + component.importForm.setValue({ file: { name: "x.ipynb" } as NzUploadFile, model: "gpt-4" }); + + vi.useFakeTimers(); + try { + vi.setSystemTime(new Date(0)); + const submitting = component.onSubmit(); + expect(component.formattedElapsedTime).toBe("0:00"); + + // Advancing the clock (even if the interval were throttled) yields the correct elapsed time. + vi.advanceTimersByTime(75_000); + expect(component.formattedElapsedTime).toBe("1:15"); + + resolveRequest(false); + await submitting; + expect((component as any).timerHandle).toBeNull(); + } finally { + vi.useRealTimers(); + } + }); + + it("has a visibilitychange handler that is safe to call (repaint is driven by the zone event)", async () => { + await createWith(of([{ name: "gpt-4" }])); + expect(() => component.onVisibilityChange()).not.toThrow(); + }); + + it("clears the stopwatch interval on destroy", async () => { + await createWith(of([{ name: "gpt-4" }])); + const clearSpy = vi.spyOn(globalThis, "clearInterval"); + (component as any).timerHandle = setInterval(() => {}, 1000); + + component.ngOnDestroy(); + + expect(clearSpy).toHaveBeenCalled(); + expect((component as any).timerHandle).toBeNull(); + }); + it("ignores a second submit while the first is still pending", async () => { // A pending requestImport models the opener still showing its overwrite confirmation. let resolveRequest!: (proceed: boolean) => void; diff --git a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.ts b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.ts index 70ee5745d1e..7a47da759fa 100644 --- a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.ts +++ b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Component, inject } from "@angular/core"; +import { Component, HostListener, inject, OnDestroy } from "@angular/core"; import { FormBuilder, FormGroup, Validators, ReactiveFormsModule } from "@angular/forms"; import { NZ_MODAL_DATA, NzModalRef } from "ng-zorro-antd/modal"; import { NzUploadComponent, NzUploadFile } from "ng-zorro-antd/upload"; @@ -25,26 +25,25 @@ import { Observable } from "rxjs"; import { AsyncPipe, NgIf, NgFor, NgOptimizedImage } from "@angular/common"; import { NzFormModule } from "ng-zorro-antd/form"; import { NzSelectModule } from "ng-zorro-antd/select"; -import { NzAlertModule } from "ng-zorro-antd/alert"; +import { NzSpinComponent } from "ng-zorro-antd/spin"; import { NzButtonComponent } from "ng-zorro-antd/button"; import { NzIconDirective } from "ng-zorro-antd/icon"; import { NotebookMigrationService } from "../../service/notebook-migration/notebook-migration.service"; -// Passed in via nzData. The modal delegates "may I proceed?" to the opener so the -// opener can keep the overwrite-confirm and generation logic (and the workflow state it -// needs) without the modal knowing about them. Resolve true to close the modal (the -// import has started), false to keep it open with the user's selection intact. +// Passed in via nzData. The modal hands the selected file and model to the opener, which runs the +// generation and persistence. The modal shows a loading state while this promise is pending. +// Resolve true to close the modal (generation succeeded and the opener navigated away), false to +// keep it open with the user's selection intact (bad file or a failure the user can retry). export interface NotebookImportModalData { requestImport: (file: NzUploadFile, model: string) => Promise; } /** * The "AI Generate Workflow from Python Notebook" modal body. It owns the upload form and - * the three model-dropdown states (loading / has models / none). On Submit it delegates the - * decision to proceed to its opener via the requestImport callback (passed in through - * nzData); the opener runs the overwrite-confirm and generation pipeline and the modal - * closes itself only when that resolves true. Mirrors the component-as-nzContent pattern - * used by the other modals opened from the menu (ResultExportationComponent, ...). + * the three model-dropdown states (loading / has models / none). On Submit it hands the file + * and model to its opener via the requestImport callback (passed in through nzData) and shows a + * loading state while generation runs, closing itself only when that resolves true. Mirrors the + * component-as-nzContent pattern used by the other modals opened from the menu. */ @Component({ selector: "texera-notebook-import-modal", @@ -58,13 +57,13 @@ export interface NotebookImportModalData { ReactiveFormsModule, NzFormModule, NzSelectModule, - NzAlertModule, + NzSpinComponent, NzUploadComponent, NzButtonComponent, NzIconDirective, ], }) -export class NotebookImportModalComponent { +export class NotebookImportModalComponent implements OnDestroy { private readonly fb = inject(FormBuilder); private readonly modalRef = inject(NzModalRef); private readonly notebookMigrationService = inject(NotebookMigrationService); @@ -90,24 +89,63 @@ export class NotebookImportModalComponent { this.modalRef.close(); } - // Guards against a second submit while the opener callback (which may show an - // overwrite confirmation) is still pending, so a double-click cannot start two imports. + // True while generation is running. Guards against a second submit and drives the loading + // spinner shown over the form. public isSubmitting = false; + private startTime: number | null = null; + private timerHandle: ReturnType | null = null; + + public ngOnDestroy(): void { + this.stopTimer(); + } + + public get formattedElapsedTime(): string { + const diffMs = this.startTime === null ? 0 : Date.now() - this.startTime; + const totalSeconds = Math.floor(diffMs / 1000); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return `${minutes}:${seconds.toString().padStart(2, "0")}`; + } + + // Empty on purpose: with default change detection, the visibilitychange event firing in the + // Angular zone is itself what triggers a repaint, so the stopwatch catches up immediately when + // the user returns to a backgrounded tab. The handler body is irrelevant. + @HostListener("document:visibilitychange") + public onVisibilityChange(): void {} + + private startTimer(): void { + this.stopTimer(); + this.startTime = Date.now(); + // The interval body is intentionally empty: the elapsed value is computed from startTime, and + // with default change detection the zone-patched timer firing is what repaints it each second. + this.timerHandle = setInterval(() => {}, 1000); + } + + private stopTimer(): void { + if (this.timerHandle !== null) { + clearInterval(this.timerHandle); + this.timerHandle = null; + } + } public async onSubmit(): Promise { if (this.isSubmitting || !this.importForm.valid) return; const file: NzUploadFile = this.importForm.get("file")?.value; const model: string = this.importForm.get("model")?.value; this.isSubmitting = true; + this.startTimer(); + this.modalRef.updateConfig({ nzClosable: false, nzMaskClosable: false, nzKeyboard: false }); try { - // Ask the opener whether to proceed; close only if it does, so cancelling the - // overwrite-confirm leaves this modal open with the selection preserved. + // Run generation via the opener; close only if it succeeds, so a failure leaves this modal + // open with the selection preserved. if (await this.data.requestImport(file, model)) { this.modalRef.close(); + return; } + this.modalRef.updateConfig({ nzClosable: true, nzMaskClosable: true, nzKeyboard: true }); } finally { - // Re-enable submit if the modal is still open (import declined or it threw). this.isSubmitting = false; + this.stopTimer(); } } } diff --git a/frontend/src/app/workspace/component/workspace.component.spec.ts b/frontend/src/app/workspace/component/workspace.component.spec.ts index f85294e42ad..588087fba86 100644 --- a/frontend/src/app/workspace/component/workspace.component.spec.ts +++ b/frontend/src/app/workspace/component/workspace.component.spec.ts @@ -106,6 +106,7 @@ describe("WorkspaceComponent", () => { disableWorkflowModification: vi.fn(), enableWorkflowModification: vi.fn(), reloadWorkflow: vi.fn(), + autoLayoutWorkflow: vi.fn(), setNewSharedModel: vi.fn(), setWorkflowMetadata: vi.fn(), clearWorkflow: vi.fn(), @@ -253,7 +254,7 @@ describe("WorkspaceComponent", () => { await createFixture(configureRoute({ id: "42" })); fixture.detectChanges(); expect(workflowActionService.setNewSharedModel).toHaveBeenCalledWith(42, { uid: 7 }); - expect(workflowActionService.reloadWorkflow).toHaveBeenCalledWith(stubWorkflow); + expect(workflowActionService.reloadWorkflow).toHaveBeenCalledWith(stubWorkflow, undefined); expect(undoRedoService.clearUndoStack).toHaveBeenCalled(); expect(undoRedoService.clearRedoStack).toHaveBeenCalled(); expect(component.isLoading).toBe(false); @@ -283,7 +284,22 @@ describe("WorkspaceComponent", () => { fixture.detectChanges(); expect(notificationService.error).toHaveBeenCalledWith(expect.stringContaining("broken")); // Workflow still flows through reload — the error is informational, not blocking. - expect(workflowActionService.reloadWorkflow).toHaveBeenCalledWith(brokenWorkflow); + expect(workflowActionService.reloadWorkflow).toHaveBeenCalledWith(brokenWorkflow, undefined); + }); + + it("with autolayout=1: renders synchronously and lays the workflow out once", async () => { + await createFixture(configureRoute({ id: "42" }, { autolayout: "1" })); + fixture.detectChanges(); + // asyncRendering=false so the operators exist in the graph before layout runs. + expect(workflowActionService.reloadWorkflow).toHaveBeenCalledWith(stubWorkflow, false); + expect(workflowActionService.autoLayoutWorkflow).toHaveBeenCalledTimes(1); + }); + + it("without autolayout: uses the default rendering and does not lay out", async () => { + await createFixture(configureRoute({ id: "42" })); + fixture.detectChanges(); + expect(workflowActionService.reloadWorkflow).toHaveBeenCalledWith(stubWorkflow, undefined); + expect(workflowActionService.autoLayoutWorkflow).not.toHaveBeenCalled(); }); it("when URL fragment matches an element in the graph, highlights it", async () => { diff --git a/frontend/src/app/workspace/component/workspace.component.ts b/frontend/src/app/workspace/component/workspace.component.ts index 2f95ccccfae..eff95ff4e6c 100644 --- a/frontend/src/app/workspace/component/workspace.component.ts +++ b/frontend/src/app/workspace/component/workspace.component.ts @@ -259,9 +259,17 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { this.workflowActionService.setNewSharedModel(wid, this.userService.getCurrentUser()); // remember URL fragment const fragment = this.route.snapshot.fragment; - // load the fetched workflow - this.workflowActionService.reloadWorkflow(workflow); + // A freshly AI-generated workflow arrives with autolayout=1: there was no canvas on the + // dashboard to lay the operators out on, so render synchronously (asyncRendering = false) + // so the operators exist in the graph, then tidy the layout once. + const shouldAutoLayout = this.route.snapshot.queryParams.autolayout === "1"; + // load the fetched workflow (asyncRendering = false for autolayout so the operators + // exist synchronously before the layout runs; undefined otherwise uses the config default) + this.workflowActionService.reloadWorkflow(workflow, shouldAutoLayout ? false : undefined); this.workflowActionService.enableWorkflowModification(); + if (shouldAutoLayout) { + this.workflowActionService.autoLayoutWorkflow(); + } // set the URL fragment to previous value // because reloadWorkflow will highlight/unhighlight all elements // which will change the URL fragment diff --git a/frontend/src/app/workspace/service/jupyter-panel/jupyter-panel.service.ts b/frontend/src/app/workspace/service/jupyter-panel/jupyter-panel.service.ts index c4f8ffcfbbc..6a14b98efed 100644 --- a/frontend/src/app/workspace/service/jupyter-panel/jupyter-panel.service.ts +++ b/frontend/src/app/workspace/service/jupyter-panel/jupyter-panel.service.ts @@ -25,7 +25,7 @@ import { HttpClient, HttpHeaders } from "@angular/common/http"; import { NotificationService } from "src/app/common/service/notification/notification.service"; import { distinctUntilChanged, switchMap } from "rxjs/operators"; import { AppSettings } from "../../../common/app-setting"; -import { NotebookMigrationService } from "../notebook-migration/notebook-migration.service"; +import { NotebookMigrationService, notebookMappingKey } from "../notebook-migration/notebook-migration.service"; import { GuiConfigService } from "../../../common/service/gui-config.service"; @Injectable({ @@ -137,7 +137,7 @@ export class JupyterPanelService { switchMap(async (response: any) => { // Only load mapping and workflow if they exist if (response.exists) { - this.notebookMigrationService.setMapping("mapping_wid_" + workflowID, response.mapping); + this.notebookMigrationService.setMapping(notebookMappingKey(workflowID), response.mapping); if ((await this.notebookMigrationService.sendNotebookToJupyter(response.notebook)) == 1) { return 1; @@ -166,7 +166,7 @@ export class JupyterPanelService { console.warn("Workflow ID is undefined. Cannot compute highlight mapping."); return; } - const mappingKey = "mapping_wid_" + wid; + const mappingKey = notebookMappingKey(wid); const mapping = this.notebookMigrationService.getMapping(mappingKey); if (mapping == undefined) { @@ -253,7 +253,7 @@ export class JupyterPanelService { this.jupyterNotebookPanelVisible.next(false); const wid = this.workflowActionService.getWorkflow().wid; if (wid != undefined) { - this.notebookMigrationService.deleteMapping("mapping_wid_" + wid); + this.notebookMigrationService.deleteMapping(notebookMappingKey(wid)); } } @@ -285,7 +285,7 @@ export class JupyterPanelService { public openJupyterNotebookPanel(): void { if (!this.enabled) return; const wid = this.workflowActionService.getWorkflow().wid; - const mappingKey = "mapping_wid_" + wid; + const mappingKey = notebookMappingKey(wid); // Check if there is corresponding mapping data if (wid === undefined || !this.notebookMigrationService.hasMapping(mappingKey)) { this.notificationService.warning("No Jupyter notebook associated with this workflow."); @@ -359,7 +359,7 @@ export class JupyterPanelService { return; } - const mappingKey = "mapping_wid_" + wid; + const mappingKey = notebookMappingKey(wid); const mappingEntry = this.notebookMigrationService.getMapping(mappingKey); if (!mappingEntry) { diff --git a/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.spec.ts b/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.spec.ts index 6570e47a00d..2b47325fdbc 100644 --- a/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.spec.ts +++ b/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.spec.ts @@ -18,7 +18,7 @@ */ import { TestBed } from "@angular/core/testing"; -import { NotebookMigrationService } from "./notebook-migration.service"; +import { NotebookMigrationService, notebookMappingKey } from "./notebook-migration.service"; import { HttpClient } from "@angular/common/http"; import { HttpClientTestingModule, HttpTestingController } from "@angular/common/http/testing"; import { NotificationService } from "src/app/common/service/notification/notification.service"; @@ -220,15 +220,21 @@ describe("NotebookMigrationService", () => { }); // storeNotebookAndMapping - it("should call storeNotebookAndMapping API", () => { - service.storeNotebookAndMapping(1, 1, {}, {}).subscribe(); + it("should call storeNotebookAndMapping API with the default vid", () => { + service.storeNotebookAndMapping(1, {}, {}).subscribe(); const req = httpMock.expectOne(req => req.url.includes("/notebook-migration/store-notebook-and-mapping")); expect(req.request.method).toBe("POST"); + expect(req.request.body.wid).toBe(1); + expect(req.request.body.vid).toBe(1); req.flush({ success: true, message: "stored" }); }); + it("notebookMappingKey builds the in-memory cache key from the wid", () => { + expect(notebookMappingKey(42)).toBe("mapping_wid_42"); + }); + // deleteNotebookAndMapping it("should call deleteNotebookAndMapping API with the wid", () => { let result: any; @@ -332,7 +338,7 @@ describe("NotebookMigrationService", () => { }); it("storeNotebookAndMapping emits a disabled result without making an HTTP call", async () => { - const result = await firstValueFrom(service.storeNotebookAndMapping(1, 1, {}, {})); + const result = await firstValueFrom(service.storeNotebookAndMapping(1, {}, {})); expect(result.success).toBe(false); httpMock.expectNone(req => req.url.includes("/notebook-migration/store-notebook-and-mapping")); }); @@ -343,4 +349,50 @@ describe("NotebookMigrationService", () => { httpMock.expectNone(req => req.url.includes("/notebook-migration/delete-notebook-and-mapping")); }); }); + + // parseAndTagNotebook (reads + validates + uuid-tags an uploaded .ipynb) + describe("parseAndTagNotebook", () => { + it("parses a valid notebook and tags every cell with a uuid", async () => { + const notebook = { + cells: [ + { cell_type: "code", source: "print(1)", metadata: {} }, + // No metadata: the parser must create it before setting the uuid. + { cell_type: "markdown", source: "# title" }, + ], + }; + const file = new File([JSON.stringify(notebook)], "analysis.ipynb"); + + const parsed = await service.parseAndTagNotebook(file); + + expect(parsed.cells.length).toBe(2); + expect(parsed.cells[0].metadata?.uuid).toBeTruthy(); + expect(parsed.cells[1].metadata?.uuid).toBeTruthy(); + }); + + it("rejects when the notebook is not valid JSON", async () => { + const file = new File(["not json"], "x.ipynb"); + await expect(service.parseAndTagNotebook(file)).rejects.toThrow(); + }); + + it("rejects a notebook without a cells array", async () => { + const file = new File([JSON.stringify({ nbformat: 4 })], "x.ipynb"); + await expect(service.parseAndTagNotebook(file)).rejects.toThrow(/Invalid notebook structure/); + }); + + it("rejects when the file content is not a string", async () => { + vi.spyOn(FileReader.prototype, "readAsText").mockImplementation(function (this: any) { + // result is a getter-only property, so shadow it with an own non-string value. + Object.defineProperty(this, "result", { value: null, configurable: true }); + this.onload?.(new ProgressEvent("load")); + }); + await expect(service.parseAndTagNotebook(new File([""], "x.ipynb"))).rejects.toThrow(/not a valid string/); + }); + + it("rejects when the file cannot be read", async () => { + vi.spyOn(FileReader.prototype, "readAsText").mockImplementation(function (this: any) { + this.onerror?.(new ProgressEvent("error")); + }); + await expect(service.parseAndTagNotebook(new File([""], "x.ipynb"))).rejects.toThrow(/Failed to read/); + }); + }); }); diff --git a/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.ts b/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.ts index 3d6600c2a2e..5a4c02af8ca 100644 --- a/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.ts +++ b/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.ts @@ -24,7 +24,9 @@ import { HttpClient, HttpHeaders } from "@angular/common/http"; import { NotificationService } from "src/app/common/service/notification/notification.service"; import { GuiConfigService } from "../../../common/service/gui-config.service"; import { WorkflowUtilService } from "../workflow-graph/util/workflow-util.service"; +import { WorkflowContent } from "../../../common/type/workflow"; import { catchError, firstValueFrom, map, Observable, of } from "rxjs"; +import { v4 as uuidv4 } from "uuid"; interface LiteLLMModel { id: string; @@ -38,7 +40,7 @@ interface LiteLLMModelsResponse { object: string; } -interface MappingContent { +export interface MappingContent { cell_to_operator: Record; operator_to_cell: Record; } @@ -54,6 +56,12 @@ interface DeleteNotebookResponse { message?: string; } +// Single source of truth for the in-memory mapping cache key. Both the dashboard generate flow +// and JupyterPanelService key the cell <-> operator mapping by this, so it must not drift. +export function notebookMappingKey(wid: number | undefined): string { + return "mapping_wid_" + wid; +} + @Injectable({ providedIn: "root", }) @@ -86,7 +94,10 @@ export class NotebookMigrationService { ); } - public async sendToAIGenerateWorkflow(notebookContent: Notebook, modelType: string) { + public async sendToAIGenerateWorkflow( + notebookContent: Notebook, + modelType: string + ): Promise<{ workflowContent: WorkflowContent; mappingContent: MappingContent }> { if (!this.enabled) throw new Error("Notebook migration feature is disabled"); const migrationLLM = this.createMigrationLLM(); // initialize() defaults to the user's Texera JWT via AuthService.getAccessToken(). @@ -195,9 +206,9 @@ export class NotebookMigrationService { public storeNotebookAndMapping( wid: number | undefined, - vid: number = 1, mappingContent: any, - notebookContent: any + notebookContent: any, + vid: number = 1 ): Observable { if (!this.enabled) { return of({ success: false, message: "Notebook migration feature is disabled" }); @@ -246,4 +257,36 @@ export class NotebookMigrationService { public deleteMapping(id: string): void { delete this.mapping[id]; } + + // Reads an uploaded .ipynb file, parses it, validates its structure, and tags each cell with a + // uuid (the cell <-> operator mapping keys off these). Rejects on a read error, invalid JSON, or + // a notebook without a cells array. Uses FileReader rather than file.text() because jsdom (the + // test environment) does not implement Blob/File.text(). + public parseAndTagNotebook(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onerror = () => reject(new Error("Failed to read the notebook file.")); + reader.onload = () => { + try { + if (typeof reader.result !== "string") { + throw new Error("File content is not a valid string."); + } + const notebook = JSON.parse(reader.result) as Notebook; + if (!notebook || !Array.isArray(notebook.cells)) { + throw new Error("Invalid notebook structure."); + } + for (const cell of notebook.cells) { + if (!cell.metadata) { + cell.metadata = {}; + } + cell.metadata.uuid = uuidv4(); + } + resolve(notebook); + } catch (error) { + reject(error); + } + }; + reader.readAsText(file); + }); + } } From 3cfee80ca1165570eeae6e5ab45a9614a11f0acd Mon Sep 17 00:00:00 2001 From: Ryan Zhang Date: Thu, 13 Aug 2026 11:02:50 -0700 Subject: [PATCH 02/10] fix(python-notebook-migration, frontend): persist the auto-layout of generated workflows --- .../src/app/workspace/component/workspace.component.spec.ts | 6 ++++++ frontend/src/app/workspace/component/workspace.component.ts | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/workspace/component/workspace.component.spec.ts b/frontend/src/app/workspace/component/workspace.component.spec.ts index 588087fba86..a17d2dbff28 100644 --- a/frontend/src/app/workspace/component/workspace.component.spec.ts +++ b/frontend/src/app/workspace/component/workspace.component.spec.ts @@ -289,10 +289,16 @@ describe("WorkspaceComponent", () => { it("with autolayout=1: renders synchronously and lays the workflow out once", async () => { await createFixture(configureRoute({ id: "42" }, { autolayout: "1" })); + const registerSpy = vi.spyOn(component, "registerAutoPersistWorkflow"); fixture.detectChanges(); // asyncRendering=false so the operators exist in the graph before layout runs. expect(workflowActionService.reloadWorkflow).toHaveBeenCalledWith(stubWorkflow, false); expect(workflowActionService.autoLayoutWorkflow).toHaveBeenCalledTimes(1); + // Auto-persistence must be registered before the layout runs, otherwise the layout's + // position-change events fire into no subscriber and the tidied layout is never saved. + expect(registerSpy.mock.invocationCallOrder[0]).toBeLessThan( + workflowActionService.autoLayoutWorkflow.mock.invocationCallOrder[0] + ); }); it("without autolayout: uses the default rendering and does not lay out", async () => { diff --git a/frontend/src/app/workspace/component/workspace.component.ts b/frontend/src/app/workspace/component/workspace.component.ts index eff95ff4e6c..1b4380a688f 100644 --- a/frontend/src/app/workspace/component/workspace.component.ts +++ b/frontend/src/app/workspace/component/workspace.component.ts @@ -267,6 +267,10 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { // exist synchronously before the layout runs; undefined otherwise uses the config default) this.workflowActionService.reloadWorkflow(workflow, shouldAutoLayout ? false : undefined); this.workflowActionService.enableWorkflowModification(); + // Register auto-persistence before autoLayoutWorkflow() runs: workflowChanged() merges hot, + // non-replaying streams, so the layout's position-change events would be lost if we subscribed + // afterwards, and the tidied layout would never be saved (autolayout=1 is one-shot). + this.registerAutoPersistWorkflow(); if (shouldAutoLayout) { this.workflowActionService.autoLayoutWorkflow(); } @@ -292,7 +296,6 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { this.undoRedoService.clearUndoStack(); this.undoRedoService.clearRedoStack(); this.setLoadingState(false); - this.registerAutoPersistWorkflow(); this.triggerCenter(); }, () => { From da6da352f15c9201ed202d66a4c875b07cb14527 Mon Sep 17 00:00:00 2001 From: Ryan Zhang Date: Thu, 13 Aug 2026 11:08:16 -0700 Subject: [PATCH 03/10] docs(python-notebook-migration, frontend): trim verbose comments --- .../user-workflow/user-workflow.component.ts | 24 +++++------------ .../notebook-import-modal.component.scss | 5 ++-- .../notebook-import-modal.component.ts | 27 +++++++------------ .../component/workspace.component.ts | 12 +++------ .../notebook-migration.service.ts | 10 +++---- 5 files changed, 26 insertions(+), 52 deletions(-) diff --git a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts index c90c595329e..ab2904425a7 100644 --- a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts +++ b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts @@ -326,10 +326,7 @@ export class UserWorkflowComponent implements AfterViewInit { return this.config.env.pythonNotebookMigrationEnabled; } - /** - * Open the AI-generate import modal from the dashboard. The modal collects the notebook file and - * model and shows a loading state while generation runs (the requestImport callback below). - */ + /** Open the AI-generate import modal, wiring its submit to generateWorkflowFromNotebook. */ public openAiGenerateModal(): void { this.modalService.create({ nzTitle: "AI Generate Workflow from Python Notebook", @@ -344,12 +341,8 @@ export class UserWorkflowComponent implements AfterViewInit { } /** - * Generate a workflow from the uploaded notebook and open it. Runs entirely on the dashboard while - * the modal shows a loading state: parse the notebook, send it to the LLM, save the result as a new - * workflow, store the notebook and cell mapping, then navigate to the new workflow. The workspace - * lays the generated operators out (via the autolayout query param) and opens the notebook panel - * (driven by the workflow id). Resolves true so the modal closes on success, or false (leaving the - * modal open with the selection intact) when the file is not a notebook or generation fails. + * Parse the notebook, generate a workflow via the LLM, save it, store the cell mapping, and open it. + * Resolves true on success (modal closes), false to keep the modal open on a bad file or a failure. */ private async generateWorkflowFromNotebook(file: NzUploadFile, model: string): Promise { const fileExtension = file.name.split(".").pop()?.toLowerCase(); @@ -375,8 +368,8 @@ export class UserWorkflowComponent implements AfterViewInit { return false; } - // Create the workflow. This is the commit point: persisting captures the expensive LLM - // result. If it fails nothing was created, so returning false (letting the user retry) is safe. + // Commit point: persisting captures the expensive LLM result. On failure nothing was created, + // so returning false to let the user retry is safe. let wid: number; try { const createdWorkflow = await firstValueFrom( @@ -395,9 +388,7 @@ export class UserWorkflowComponent implements AfterViewInit { return false; } - // Past the commit point the follow-up steps are best-effort: a transient failure must not - // discard the created workflow or the LLM result, so we log/warn and still open the workflow - // rather than force a full re-generation. + // Best-effort follow-ups: never discard the created workflow, so log/warn and still open it. if (this.pid) { try { await firstValueFrom(this.userProjectService.addWorkflowToProject(this.pid, wid)); @@ -425,8 +416,7 @@ export class UserWorkflowComponent implements AfterViewInit { return true; } - // Strips the extension from an uploaded file name, falling back to DEFAULT_WORKFLOW_NAME when the - // result is empty. Shared by the file-upload and AI-generate flows to name the created workflow. + // Strips the extension from a file name, falling back to DEFAULT_WORKFLOW_NAME when empty. private deriveWorkflowName(fileName: string): string { const extensionIndex = fileName.lastIndexOf("."); const baseName = extensionIndex === -1 ? fileName : fileName.substring(0, extensionIndex); diff --git a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.scss b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.scss index d5d60d1a345..da51cba51ee 100644 --- a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.scss +++ b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.scss @@ -70,8 +70,7 @@ width: 50%; } - // Wraps the form and footer so the loading overlay can cover them without changing the - // modal's height (which would otherwise resize and re-center the centered modal on submit). + // Positioning context for the loading overlay, so covering the form does not resize the modal. &-content { position: relative; } @@ -80,7 +79,7 @@ position: absolute; inset: 0; z-index: 1; - // Solid background so the form and footer underneath are fully hidden. + // Solid background so the form underneath is fully hidden. background-color: #fff; display: flex; flex-direction: column; diff --git a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.ts b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.ts index 7a47da759fa..7c9ad86cd66 100644 --- a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.ts +++ b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.ts @@ -30,20 +30,15 @@ import { NzButtonComponent } from "ng-zorro-antd/button"; import { NzIconDirective } from "ng-zorro-antd/icon"; import { NotebookMigrationService } from "../../service/notebook-migration/notebook-migration.service"; -// Passed in via nzData. The modal hands the selected file and model to the opener, which runs the -// generation and persistence. The modal shows a loading state while this promise is pending. -// Resolve true to close the modal (generation succeeded and the opener navigated away), false to -// keep it open with the user's selection intact (bad file or a failure the user can retry). +// Passed in via nzData. requestImport resolves true to close the modal, false to keep it open +// with the user's selection intact (bad file or a retryable failure). export interface NotebookImportModalData { requestImport: (file: NzUploadFile, model: string) => Promise; } /** - * The "AI Generate Workflow from Python Notebook" modal body. It owns the upload form and - * the three model-dropdown states (loading / has models / none). On Submit it hands the file - * and model to its opener via the requestImport callback (passed in through nzData) and shows a - * loading state while generation runs, closing itself only when that resolves true. Mirrors the - * component-as-nzContent pattern used by the other modals opened from the menu. + * The "AI Generate Workflow from Python Notebook" modal body: the upload form and model dropdown. + * On Submit it hands the file and model to requestImport and shows a loading state until it resolves. */ @Component({ selector: "texera-notebook-import-modal", @@ -89,8 +84,7 @@ export class NotebookImportModalComponent implements OnDestroy { this.modalRef.close(); } - // True while generation is running. Guards against a second submit and drives the loading - // spinner shown over the form. + // True while generation runs: guards against a second submit and drives the loading overlay. public isSubmitting = false; private startTime: number | null = null; private timerHandle: ReturnType | null = null; @@ -107,17 +101,15 @@ export class NotebookImportModalComponent implements OnDestroy { return `${minutes}:${seconds.toString().padStart(2, "0")}`; } - // Empty on purpose: with default change detection, the visibilitychange event firing in the - // Angular zone is itself what triggers a repaint, so the stopwatch catches up immediately when - // the user returns to a backgrounded tab. The handler body is irrelevant. + // Empty body on purpose: the zone-patched event firing is itself what repaints the stopwatch, + // so it catches up when the user returns to a backgrounded tab. Same reason as the timer below. @HostListener("document:visibilitychange") public onVisibilityChange(): void {} private startTimer(): void { this.stopTimer(); this.startTime = Date.now(); - // The interval body is intentionally empty: the elapsed value is computed from startTime, and - // with default change detection the zone-patched timer firing is what repaints it each second. + // Empty body: elapsed is computed from startTime; the zone-patched tick just triggers a repaint. this.timerHandle = setInterval(() => {}, 1000); } @@ -136,8 +128,7 @@ export class NotebookImportModalComponent implements OnDestroy { this.startTimer(); this.modalRef.updateConfig({ nzClosable: false, nzMaskClosable: false, nzKeyboard: false }); try { - // Run generation via the opener; close only if it succeeds, so a failure leaves this modal - // open with the selection preserved. + // Close only on success, so a failure leaves the modal open with the selection preserved. if (await this.data.requestImport(file, model)) { this.modalRef.close(); return; diff --git a/frontend/src/app/workspace/component/workspace.component.ts b/frontend/src/app/workspace/component/workspace.component.ts index 1b4380a688f..b451527498b 100644 --- a/frontend/src/app/workspace/component/workspace.component.ts +++ b/frontend/src/app/workspace/component/workspace.component.ts @@ -259,17 +259,13 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { this.workflowActionService.setNewSharedModel(wid, this.userService.getCurrentUser()); // remember URL fragment const fragment = this.route.snapshot.fragment; - // A freshly AI-generated workflow arrives with autolayout=1: there was no canvas on the - // dashboard to lay the operators out on, so render synchronously (asyncRendering = false) - // so the operators exist in the graph, then tidy the layout once. + // An AI-generated workflow arrives with autolayout=1. Render synchronously + // (asyncRendering = false) so the operators exist before the one-shot layout runs. const shouldAutoLayout = this.route.snapshot.queryParams.autolayout === "1"; - // load the fetched workflow (asyncRendering = false for autolayout so the operators - // exist synchronously before the layout runs; undefined otherwise uses the config default) this.workflowActionService.reloadWorkflow(workflow, shouldAutoLayout ? false : undefined); this.workflowActionService.enableWorkflowModification(); - // Register auto-persistence before autoLayoutWorkflow() runs: workflowChanged() merges hot, - // non-replaying streams, so the layout's position-change events would be lost if we subscribed - // afterwards, and the tidied layout would never be saved (autolayout=1 is one-shot). + // Register before autoLayoutWorkflow(): workflowChanged() streams are hot, so subscribing + // afterward would drop the layout's position events and the tidied layout would never save. this.registerAutoPersistWorkflow(); if (shouldAutoLayout) { this.workflowActionService.autoLayoutWorkflow(); diff --git a/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.ts b/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.ts index 5a4c02af8ca..5c636240f36 100644 --- a/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.ts +++ b/frontend/src/app/workspace/service/notebook-migration/notebook-migration.service.ts @@ -56,8 +56,7 @@ interface DeleteNotebookResponse { message?: string; } -// Single source of truth for the in-memory mapping cache key. Both the dashboard generate flow -// and JupyterPanelService key the cell <-> operator mapping by this, so it must not drift. +// Single source of truth for the mapping cache key, shared with JupyterPanelService so it can't drift. export function notebookMappingKey(wid: number | undefined): string { return "mapping_wid_" + wid; } @@ -258,10 +257,9 @@ export class NotebookMigrationService { delete this.mapping[id]; } - // Reads an uploaded .ipynb file, parses it, validates its structure, and tags each cell with a - // uuid (the cell <-> operator mapping keys off these). Rejects on a read error, invalid JSON, or - // a notebook without a cells array. Uses FileReader rather than file.text() because jsdom (the - // test environment) does not implement Blob/File.text(). + // Reads and parses an .ipynb file, then tags each cell with a uuid (the mapping keys off these). + // Rejects on a read error, invalid JSON, or a missing cells array. Uses FileReader rather than + // file.text() because jsdom (the test environment) does not implement Blob/File.text(). public parseAndTagNotebook(file: File): Promise { return new Promise((resolve, reject) => { const reader = new FileReader(); From d3c72f6382b0d626f58b2421e4b3326edbdea825 Mon Sep 17 00:00:00 2001 From: Ryan Zhang Date: Thu, 13 Aug 2026 11:24:19 -0700 Subject: [PATCH 04/10] fix(python-notebook-migration, frontend): make the modal loading overlay accessible --- .../notebook-import-modal.component.html | 9 +++++++-- .../notebook-import-modal.component.spec.ts | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.html b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.html index 4edfc7dbe02..3986eb9c710 100644 --- a/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.html +++ b/frontend/src/app/workspace/component/notebook-import-modal/notebook-import-modal.component.html @@ -29,6 +29,7 @@

@@ -113,7 +114,9 @@

-