[RHDHBUGS- 3302] using Markitdown for AI notebooks

workspaces/intelligent-assistant/.changeset/fuzzy-peaches-shake.md

@@ -0,0 +1,5 @@
+---
+'@red-hat-developer-hub/backstage-plugin-lightspeed-backend': minor
+---
+
+bugfix - Notebooks routes 404 passthrough error resolved

workspaces/intelligent-assistant/.changeset/six-chairs-turn.md

@@ -0,0 +1,5 @@
+---
+'@red-hat-developer-hub/backstage-plugin-intelligent-assistant-backend': minor
+---
+
+AI Notebooks will use markitdown to clean up data before vectorizing documents

workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/package.json

@@ -54,20 +54,17 @@
     "express": "^4.21.1",
     "express-rate-limit": "^8.2.2",
     "form-data": "^4.0.6",
-    "htmlparser2": "^9.1.0",
     "http-proxy-middleware": "^3.0.7",
-    "js-yaml": "^4.1.1",
     "knex": "^3.1.0",
-    "multer": "^2.2.0",
-    "pdfjs-dist": "^4.10.38"
+    "markitdown-ts": "^0.0.10",
+    "multer": "^2.2.0"
   },
   "devDependencies": {
     "@backstage/backend-test-utils": "^1.11.4",
     "@backstage/cli": "^0.36.3",
     "@ianvs/prettier-plugin-sort-imports": "^4.4.0",
     "@spotify/prettier-config": "^15.0.0",
     "@types/express": "4.17.25",
-    "@types/js-yaml": "^4",
     "@types/multer": "^1.4.12",
     "@types/supertest": "2.0.16",
     "msw": "2.14.6",

workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/notebooks/documents/documentHelpers.test.ts

@@ -14,22 +14,10 @@
  * limitations under the License.
  */
 
-import { mockServices } from '@backstage/backend-test-utils';
-
 import { DEFAULT_MAX_FILE_SIZE_MB } from '../../constant';
-import {
-  isValidFileSize,
-  isValidFileType,
-  parseFileContent,
-} from './documentHelpers';
+import { isValidFileSize, isValidFileType } from './documentHelpers';
 
 describe('documentHelpers', () => {
-  const logger = mockServices.logger.mock();
-
-  afterEach(() => {
-    jest.clearAllMocks();
-  });
-
   describe('isValidFileSize', () => {
     const MB = 1024 * 1024;
 
@@ -93,38 +81,4 @@ describe('documentHelpers', () => {
       expect(isValidFileType('.')).toBe(false);
     });
   });
-
-  describe('parseFileContent', () => {
-    it('should throw error when no file uploaded for non-URL type', async () => {
-      await expect(parseFileContent(logger, 'txt', undefined)).rejects.toThrow(
-        'No file uploaded',
-      );
-    });
-
-    it('should throw error when file size exceeds limit', async () => {
-      const largeFile = {
-        buffer: Buffer.from('test'),
-        originalname: 'large.txt',
-        size: 21 * 1024 * 1024, // 21MB
-      } as Express.Multer.File;
-
-      await expect(parseFileContent(logger, 'txt', largeFile)).rejects.toThrow(
-        `File size exceeds ${DEFAULT_MAX_FILE_SIZE_MB / 1024 / 1024}MB limit`,
-      );
-    });
-
-    it('should parse valid file successfully', async () => {
-      const file = {
-        buffer: Buffer.from('test content'),
-        originalname: 'test.txt',
-        size: 1024,
-      } as Express.Multer.File;
-
-      const result = await parseFileContent(logger, 'txt', file);
-
-      expect(result.content).toBe('test content');
-      expect(result.metadata.fileName).toBe('test.txt');
-      expect(result.metadata.fileType).toBe('txt');
-    });
-  });
 });

workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/notebooks/documents/documentHelpers.ts

@@ -13,25 +13,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { LoggerService } from '@backstage/backend-plugin-api';
 import { InputError } from '@backstage/errors';
 
-import { Parser } from 'htmlparser2';
-
 import dns from 'dns/promises';
 import { isIP } from 'net';
 
 import {
   DEFAULT_MAX_FILE_SIZE_MB,
   FILTERED_CONTENT_MARKER,
-  HTML_BLOCK_TAGS,
-  HTML_IGNORED_TAGS,
   MAX_CONSECUTIVE_NEWLINES,
   PROMPT_INJECTION_PATTERNS,
   SSRF_BLOCKED_HOSTNAMES,
   SupportedFileType,
 } from '../../constant';
-import { parseFile } from './fileParser';
 
 // ==============================================================================
 // URL Support Functions (Reserved for Future URL File Type Support)
@@ -78,30 +72,6 @@ export const isValidFileType = (fileType: string): boolean => {
   );
 };
 
-/**
- * Parse file from upload
- * @param logger - Logger service
- * @param fileType - File type
- * @param file - File to parse
- * @returns Parsed file
- */
-export const parseFileContent = async (
-  logger: LoggerService,
-  fileType: string,
-  file?: Express.Multer.File,
-) => {
-  if (!file) {
-    throw new InputError('No file uploaded');
-  }
-  if (!isValidFileSize(file.size)) {
-    throw new InputError(
-      `File size exceeds ${DEFAULT_MAX_FILE_SIZE_MB / 1024 / 1024}MB limit`,
-    );
-  }
-  logger.info(`Parsing file ${file.originalname} for fileType ${fileType}`);
-  return await parseFile(file.buffer, file.originalname, fileType);
-};
-
 /**
  * Check if an IP address is private, internal, or a metadata endpoint
  * Blocks SSRF attacks to internal networks
@@ -243,39 +213,6 @@ export const validateURLForSSRF = async (urlString: string): Promise<void> => {
   }
 };
 
-/**
- * Strip HTML tags and extract readable text from HTML content
- * @reserved Reserved for future URL file type support
- */
-export const stripHtmlTags = (html: string): string => {
-  let text = '';
-  let ignoring = false;
-
-  const parser = new Parser(
-    {
-      onopentag(name) {
-        if (HTML_IGNORED_TAGS.has(name)) ignoring = true;
-      },
-      onclosetag(name) {
-        if (HTML_IGNORED_TAGS.has(name)) ignoring = false;
-        if (HTML_BLOCK_TAGS.has(name)) text += '\n';
-      },
-      ontext(data) {
-        if (!ignoring) text += data;
-      },
-    },
-    { decodeEntities: true },
-  );
-
-  parser.write(html);
-  parser.end();
-
-  return text
-    .replace(/\n\s*\n/g, '\n\n')
-    .replace(/[ \t]+/g, ' ')
-    .trim();
-};
-
 /**
  * Sanitize content to prevent prompt injection attacks
  * Detects and filters common prompt injection patterns

workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/notebooks/documents/documentService.ts

@@ -25,7 +25,6 @@ import {
 } from '../../constant';
 import { SessionDocument, UpsertResult } from '../types/notebooksTypes';
 import { VectorStoresOperator } from '../VectorStoresOperator';
-import { toFile } from './fileParser';
 
 /**
  * Service for managing documents within notebook sessions using File-Based API
@@ -92,21 +91,22 @@ export class DocumentService {
   }
 
   /**
-   * Upload a file to the Files API
-   * @param content - File content as string
+   * Upload markdown content to the Files API
+   * @param content - Markdown content string
    * @param title - File title/name
    * @returns File ID from the Files API
    * @throws Error if upload fails
    */
   async uploadFile(content: string, title: string): Promise<string> {
     try {
-      // Determine MIME type from file type or default to text/plain
-      const mimeType = 'text/plain';
-      const txtFilename = `${title.replace(/\.[^.]+$/, '')}.txt`;
+      const mdFilename = `${title.replace(/\.[^.]+$/, '')}.md`;
+
       const file = await this.client.files.create({
-        file: await toFile(Buffer.from(content, 'utf-8'), txtFilename, {
-          type: mimeType,
-        }),
+        file: {
+          name: mdFilename,
+          buffer: Buffer.from(content, 'utf-8'),
+          type: 'text/markdown',
+        },
         purpose: 'assistants',
       });
 
@@ -115,11 +115,9 @@ export class DocumentService {
       );
       return file.id;
     } catch (error) {
-      // Preserve the original error type and message
       if (error instanceof Error) {
         throw error;
       }
-      // For non-Error objects, wrap with context
       throw new Error(`Failed to upload file: ${String(error)}`);
     }
   }