diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c759340..ea3ac07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,11 +35,24 @@ jobs: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: sqlmap + ports: + - 3306:3306 + options: --health-cmd "mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent" + --health-interval 10s --health-timeout 5s --health-retries 10 env: PGPASS: postgres PGUSER: postgres PGDB: sqlmap PGHOST: localhost + MYSQLHOST: 127.0.0.1 + MYSQLUSER: root + MYSQLPASS: mysql + MYSQLDB: sqlmap steps: - name: Checkout source code uses: actions/checkout@v6 @@ -53,6 +66,8 @@ jobs: run: npm run lint - name: Test run: npm test + - name: Test Integration + run: npm run test:integration - name: Test Security run: npm run test:security automerge: diff --git a/README.md b/README.md index 4452f1d..f725729 100644 --- a/README.md +++ b/README.md @@ -289,13 +289,29 @@ const pascalOrCamelToSnake = str => This module can be tested and reported on in a variety of ways... ```sh -npm run test # runs tap based unit test suite. -npm run test:security # runs sqlmap security tests. -npm run test:typescript # runs type definition tests. -npm run coverage # generates a coverage report in docs dir. -npm run lint # lints via standardJS. +npm run test # runs the node:test unit test suite. +npm run test:integration # runs integration tests against running PostgreSQL & MySQL. +npm run test:integration:docker # spins up DBs via docker compose, runs integration tests, tears down. +npm run test:security # runs sqlmap security tests. +npm run test:typescript # runs type definition tests. +npm run coverage # generates a coverage report in docs dir. +npm run lint # lints via standardJS. ``` +The integration suite executes queries built by every public API against real +PostgreSQL and MySQL instances (the latter via both the `mysql` and `mysql2` +drivers), verifying the generated SQL is valid and that interpolated values are +stored as literals. To run it locally: + +```sh +npm run db:up # start postgres + mysql via docker compose (waits for healthy) +npm run test:integration # run the suite +npm run db:down # tear down +``` + +Connection settings default to the docker-compose services and can be overridden +with `PGHOST/PGPORT/PGUSER/PGPASS/PGDB` and `MYSQLHOST/MYSQLPORT/MYSQLUSER/MYSQLPASS/MYSQLDB`. + ## Benchmark Find more about `@nearform/sql` speed [here](benchmark) diff --git a/docker-compose.yml b/docker-compose.yml index caf1663..355eed7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,28 @@ -version: '3.4' +version: '3.9' services: - db: - image: postgres:9-alpine + postgres: + image: postgres:17-alpine environment: POSTGRES_PASSWORD: postgres POSTGRES_USER: postgres POSTGRES_DB: sqlmap ports: - 5432:5432 + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U postgres'] + interval: 5s + timeout: 5s + retries: 10 + + mysql: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: sqlmap + ports: + - 3306:3306 + healthcheck: + test: ['CMD-SHELL', 'mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent'] + interval: 5s + timeout: 5s + retries: 10 diff --git a/integration/SQL.mysql.integration.test.js b/integration/SQL.mysql.integration.test.js new file mode 100644 index 0000000..da6c3e2 --- /dev/null +++ b/integration/SQL.mysql.integration.test.js @@ -0,0 +1,18 @@ +'use strict' + +const { test, before, after } = require('node:test') +const { withMysql, createUsersTable } = require('./helpers/db') +const runFeatureSuite = require('./shared/featureSuite') + +const ctx = { db: null } + +before(async () => { + ctx.db = await withMysql() + await createUsersTable(ctx.db) +}) + +after(async () => { + if (ctx.db) await ctx.db.end() +}) + +runFeatureSuite(test, () => ctx.db) diff --git a/integration/SQL.mysql2.integration.test.js b/integration/SQL.mysql2.integration.test.js new file mode 100644 index 0000000..358e02b --- /dev/null +++ b/integration/SQL.mysql2.integration.test.js @@ -0,0 +1,18 @@ +'use strict' + +const { test, before, after } = require('node:test') +const { withMysql2, createUsersTable } = require('./helpers/db') +const runFeatureSuite = require('./shared/featureSuite') + +const ctx = { db: null } + +before(async () => { + ctx.db = await withMysql2() + await createUsersTable(ctx.db) +}) + +after(async () => { + if (ctx.db) await ctx.db.end() +}) + +runFeatureSuite(test, () => ctx.db) diff --git a/integration/SQL.pg.integration.test.js b/integration/SQL.pg.integration.test.js new file mode 100644 index 0000000..ee69126 --- /dev/null +++ b/integration/SQL.pg.integration.test.js @@ -0,0 +1,18 @@ +'use strict' + +const { test, before, after } = require('node:test') +const { withPg, createUsersTable } = require('./helpers/db') +const runFeatureSuite = require('./shared/featureSuite') + +const ctx = { db: null } + +before(async () => { + ctx.db = await withPg() + await createUsersTable(ctx.db) +}) + +after(async () => { + if (ctx.db) await ctx.db.end() +}) + +runFeatureSuite(test, () => ctx.db) diff --git a/integration/helpers/config.js b/integration/helpers/config.js new file mode 100644 index 0000000..39a44d2 --- /dev/null +++ b/integration/helpers/config.js @@ -0,0 +1,26 @@ +'use strict' + +// PostgreSQL connection config. +// Mirrors the env-var convention already used by sqlmap/config.js so the same +// CI environment variables drive both suites. +const pg = { + user: process.env.PGUSER || 'postgres', + host: process.env.PGHOST || 'localhost', + database: process.env.PGDB || 'sqlmap', + password: process.env.PGPASS || 'postgres', + port: Number(process.env.PGPORT) || 5432 +} + +// MySQL connection config, shared by both the `mysql` and `mysql2` drivers. +const mysql = { + host: process.env.MYSQLHOST || 'localhost', + port: Number(process.env.MYSQLPORT) || 3306, + user: process.env.MYSQLUSER || 'root', + password: process.env.MYSQLPASS || 'mysql', + database: process.env.MYSQLDB || 'sqlmap', + // utf8mb4 so 4-byte characters (e.g. emoji) round-trip on both the + // `mysql` driver (defaults to 3-byte utf8) and `mysql2`. + charset: 'utf8mb4' +} + +module.exports = { pg, mysql } diff --git a/integration/helpers/db.js b/integration/helpers/db.js new file mode 100644 index 0000000..d829aa4 --- /dev/null +++ b/integration/helpers/db.js @@ -0,0 +1,114 @@ +'use strict' + +const config = require('./config') + +// Each adapter exposes the same shape so the shared feature suite is +// driver-agnostic: +// - dialect: 'pg' | 'mysql' (drives dialect-specific SQL in the suite) +// - query(stmt): runs a SqlStatement, returns the result rows +// - raw(text): runs a plain SQL string (used for DDL / cleanup) +// - end(): closes the connection + +// PostgreSQL via `pg`. A SqlStatement is passed straight to client.query() +// because it exposes `.text` and `.values` getters — the documented usage. +async function withPg () { + const { Client } = require('pg') + const client = new Client(config.pg) + await client.connect() + return { + dialect: 'pg', + async query (stmt) { + const res = await client.query(stmt) + return res.rows + }, + async raw (text) { + const res = await client.query(text) + return res.rows + }, + async end () { + await client.end() + } + } +} + +// MySQL via `mysql2/promise`. The driver reads `sql` and `values` off the +// options object, which map directly onto a SqlStatement's `.sql`/`.values`. +async function withMysql2 () { + const mysql = require('mysql2/promise') + const conn = await mysql.createConnection(config.mysql) + return { + dialect: 'mysql', + async query (stmt) { + const [rows] = await conn.query({ sql: stmt.sql, values: stmt.values }) + return rows + }, + async raw (text) { + const [rows] = await conn.query(text) + return rows + }, + async end () { + await conn.end() + } + } +} + +// The legacy `mysql` driver cannot speak MySQL 8's default +// `caching_sha2_password` auth. mysql2 can, so we use it to switch the account +// to `mysql_native_password` first. This works identically locally and in CI +// without needing a container `command` override (unsupported by GitHub +// Actions service containers). +async function ensureNativePassword () { + const mysql = require('mysql2/promise') + const conn = await mysql.createConnection(config.mysql) + try { + // We connect over TCP, so the account is `@'%'`. `?` placeholders are + // escaped client-side (text protocol) into a valid `'user'@'%'` account. + await conn.query( + "ALTER USER ?@'%' IDENTIFIED WITH mysql_native_password BY ?", + [config.mysql.user, config.mysql.password] + ) + } finally { + await conn.end() + } +} + +// MySQL via the original callback-based `mysql` driver, promisified. +async function withMysql () { + await ensureNativePassword() + const mysql = require('mysql') + const conn = mysql.createConnection(config.mysql) + await new Promise((resolve, reject) => + conn.connect(err => (err ? reject(err) : resolve())) + ) + const run = (sql, values) => + new Promise((resolve, reject) => + conn.query(sql, values, (err, rows) => (err ? reject(err) : resolve(rows))) + ) + return { + dialect: 'mysql', + async query (stmt) { + return run(stmt.sql, stmt.values) + }, + async raw (text) { + return run(text) + }, + async end () { + await new Promise(resolve => conn.end(() => resolve())) + } + } +} + +// Fresh, portable `users` table. Explicit (non-auto) integer ids keep the +// inserts identical across dialects. +async function createUsersTable (db) { + await db.raw('DROP TABLE IF EXISTS users') + await db.raw(`CREATE TABLE users ( + id INTEGER PRIMARY KEY, + username VARCHAR(255) NOT NULL, + email VARCHAR(255), + password VARCHAR(255), + metadata VARCHAR(255) + )`) +} + +module.exports = { withPg, withMysql, withMysql2, createUsersTable } diff --git a/integration/shared/featureSuite.js b/integration/shared/featureSuite.js new file mode 100644 index 0000000..2dc52bf --- /dev/null +++ b/integration/shared/featureSuite.js @@ -0,0 +1,186 @@ +'use strict' + +const assert = require('node:assert') +const SQL = require('../../SQL') + +// Runs the full @nearform/sql feature matrix against a live database. +// Every case builds a query with the public API and EXECUTES it, then asserts +// on the rows read back — proving the generated SQL is both valid and safe. +// +// test - the node:test `test` function from the calling file +// getDb - returns the connected adapter (see helpers/db.js). It's a getter +// because the connection is opened in the file's `before` hook, +// after this function has registered its subtests. +module.exports = function runFeatureSuite (test, getDb) { + const reset = db => db.raw('DELETE FROM users') + + test('basic parameterized insert/select round-trips', async () => { + const db = getDb() + await reset(db) + await db.query(SQL` + INSERT INTO users (id, username, email, password) + VALUES (${1}, ${'alice'}, ${'alice@example.com'}, ${'secret'}) + `) + const rows = await db.query(SQL` + SELECT id, username, email, password FROM users WHERE id = ${1} + `) + assert.equal(rows.length, 1) + assert.equal(rows[0].username, 'alice') + assert.equal(rows[0].email, 'alice@example.com') + assert.equal(rows[0].password, 'secret') + }) + + test('interpolated values are stored as literals, not executed (injection)', async () => { + const db = getDb() + await reset(db) + const evil = "Robert'); DROP TABLE users;--" + await db.query(SQL`INSERT INTO users (id, username) VALUES (${2}, ${evil})`) + const rows = await db.query(SQL`SELECT username FROM users WHERE id = ${2}`) + assert.equal(rows[0].username, evil) + // The table must still exist and hold exactly the row we inserted. + const all = await db.query(SQL`SELECT id FROM users`) + assert.equal(all.length, 1) + }) + + test('glue builds a working IN clause', async () => { + const db = getDb() + await reset(db) + for (const id of [1, 2, 3, 4]) { + await db.query(SQL`INSERT INTO users (id, username) VALUES (${id}, ${'u' + id})`) + } + const ids = [1, 3] + const rows = await db.query(SQL` + SELECT id FROM users + WHERE id IN (${SQL.glue(ids.map(id => SQL`${id}`), ' , ')}) + ORDER BY id + `) + assert.deepEqual(rows.map(r => Number(r.id)), [1, 3]) + }) + + test('glue builds a working batch insert', async () => { + const db = getDb() + await reset(db) + const users = [ + { id: 10, name: 'u10' }, + { id: 11, name: 'u11' }, + { id: 12, name: 'u12' } + ] + await db.query(SQL` + INSERT INTO users (id, username) + VALUES ${SQL.glue(users.map(u => SQL`(${u.id}, ${u.name})`), ' , ')} + `) + const rows = await db.query(SQL`SELECT id, username FROM users ORDER BY id`) + assert.equal(rows.length, 3) + assert.deepEqual(rows.map(r => r.username), ['u10', 'u11', 'u12']) + }) + + test('map builds a working IN clause (default and object mapper)', async () => { + const db = getDb() + await reset(db) + for (const id of [21, 22, 23, 24]) { + await db.query(SQL`INSERT INTO users (id, username) VALUES (${id}, ${'u' + id})`) + } + // default mapper over an array of scalars + const rows = await db.query(SQL` + SELECT id FROM users WHERE id IN (${SQL.map([21, 23])}) ORDER BY id + `) + assert.deepEqual(rows.map(r => Number(r.id)), [21, 23]) + + // explicit mapper over an array of objects + const objs = [{ id: 22 }, { id: 24 }] + const rows2 = await db.query(SQL` + SELECT id FROM users WHERE id IN (${SQL.map(objs, o => o.id)}) ORDER BY id + `) + assert.deepEqual(rows2.map(r => Number(r.id)), [22, 24]) + }) + + test('quoteIdent produces valid quoted identifiers (incl. escaping)', async () => { + const db = getDb() + // dynamic column + table name on the existing table + await reset(db) + await db.query(SQL`INSERT INTO ${SQL.quoteIdent('users')} (id, username) VALUES (${80}, ${'quoted'})`) + const rows = await db.query(SQL` + SELECT ${SQL.quoteIdent('username')} FROM ${SQL.quoteIdent('users')} WHERE id = ${80} + `) + assert.equal(rows[0].username, 'quoted') + + // an identifier that is INVALID unquoted (contains a space) and one that + // contains the dialect's own quote char — exercises the escaping path. + const weird = db.dialect === 'pg' ? 'we"ird table' : 'we`ird table' + await db.raw(`DROP TABLE IF EXISTS ${quoteRaw(db.dialect, weird)}`) + await db.query(SQL`CREATE TABLE ${SQL.quoteIdent(weird)} (id INTEGER PRIMARY KEY)`) + try { + await db.query(SQL`INSERT INTO ${SQL.quoteIdent(weird)} (id) VALUES (${99})`) + const wrows = await db.query(SQL`SELECT id FROM ${SQL.quoteIdent(weird)} WHERE id = ${99}`) + assert.equal(Number(wrows[0].id), 99) + } finally { + await db.raw(`DROP TABLE IF EXISTS ${quoteRaw(db.dialect, weird)}`) + } + }) + + test('unsafe interpolates a trusted fragment literally', async () => { + const db = getDb() + await reset(db) + const columns = ['id', 'username'] // trusted, not user input + await db.query(SQL` + INSERT INTO users (${SQL.unsafe(columns.join(', '))}) + VALUES (${30}, ${'unsafe-user'}) + `) + const rows = await db.query(SQL`SELECT username FROM users WHERE id = ${30}`) + assert.equal(rows[0].username, 'unsafe-user') + }) + + test('nested SqlStatement keeps placeholder numbering correct', async () => { + const db = getDb() + await reset(db) + await db.query(SQL` + INSERT INTO users (id, username, email) + VALUES (${40}, ${'bob'}, ${'bob@example.com'}) + `) + // For pg this generates `... id = $1 AND (username = $2 AND email = $3)`; + // executing it proves the $N offset logic is correct against a real server. + const condition = SQL`username = ${'bob'} AND email = ${'bob@example.com'}` + const rows = await db.query(SQL` + SELECT id FROM users WHERE id = ${40} AND (${condition}) + `) + assert.equal(rows.length, 1) + assert.equal(Number(rows[0].id), 40) + }) + + test('deprecated append builds an executable query', async () => { + const db = getDb() + await reset(db) + await db.query(SQL`INSERT INTO users (id, username) VALUES (${50}, ${'appended'})`) + const sql = SQL`SELECT id, username FROM users WHERE username = ${'appended'}` + sql.append(SQL` AND id = ${50}`) + const rows = await db.query(sql) + assert.equal(rows.length, 1) + assert.equal(Number(rows[0].id), 50) + }) + + test('null values round-trip', async () => { + const db = getDb() + await reset(db) + await db.query(SQL` + INSERT INTO users (id, username, metadata) VALUES (${60}, ${'nulluser'}, ${null}) + `) + const rows = await db.query(SQL`SELECT metadata FROM users WHERE id = ${60}`) + assert.equal(rows[0].metadata, null) + }) + + test('special characters and unicode round-trip intact', async () => { + const db = getDb() + await reset(db) + const weirdVal = 'o\'brien "the\\great" 😀 ®' + await db.query(SQL`INSERT INTO users (id, username) VALUES (${70}, ${weirdVal})`) + const rows = await db.query(SQL`SELECT username FROM users WHERE id = ${70}`) + assert.equal(rows[0].username, weirdVal) + }) +} + +// Quote an identifier for raw (non-SqlStatement) DDL — same rule as +// quoteIdentifier.js, used only for setup/teardown of the "weird" table. +function quoteRaw (dialect, value) { + const q = dialect === 'mysql' ? '`' : '"' + return q + value.replace(new RegExp(q, 'g'), q + q) + q +} diff --git a/package.json b/package.json index 82cf3c6..b5223ea 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,10 @@ "scripts": { "test": "node --test *.test.js", "posttest": "tstyche", + "db:up": "docker compose up -d --wait", + "db:down": "docker compose down -v", + "test:integration": "node --test --test-concurrency=1 integration/**/*.test.js", + "test:integration:docker": "npm run db:up && npm run test:integration; npm run db:down", "test:security": "node ./sqlmap/sqlmap.js", "test:typescript": "tstyche", "pretest:security": "git clone --depth=1 https://github.com/sqlmapproject/sqlmap node_modules/sqlmap && node ./sqlmap/db-init.js", @@ -29,10 +33,12 @@ "benchmark": "^2.1.4", "fastify": "^5.8.5", "jsonfile": "^6.2.1", - "pg": "^8.20.0", + "mysql": "^2.18.1", + "mysql2": "^3.22.5", + "pg": "^8.21.0", "sql-template-strings": "^2.2.2", "standard": "^17.1.2", - "tstyche": "^7.1.0" + "tstyche": "^7.2.1" }, "standard": { "ignore": [