diff --git a/packages/pglite-initdb/.gitignore b/packages/pglite-initdb/.gitignore new file mode 100644 index 000000000..ae02570c9 --- /dev/null +++ b/packages/pglite-initdb/.gitignore @@ -0,0 +1 @@ +release/ \ No newline at end of file diff --git a/packages/pglite-initdb/CHANGELOG.md b/packages/pglite-initdb/CHANGELOG.md new file mode 100644 index 000000000..f9ed2b3ba --- /dev/null +++ b/packages/pglite-initdb/CHANGELOG.md @@ -0,0 +1,157 @@ +# @electric-sql/pglite-tools + +## 0.2.19 + +### Patch Changes + +- Updated dependencies [8785034] +- Updated dependencies [90cfee8] + - @electric-sql/pglite@0.3.14 + +## 0.2.18 + +### Patch Changes + +- ad3d0d8: Updated pg_dump to use callback data exchange; built pg_dump with emscripten +- Updated dependencies [ad3d0d8] + - @electric-sql/pglite@0.3.13 + +## 0.2.17 + +### Patch Changes + +- Updated dependencies [ce0e74e] + - @electric-sql/pglite@0.3.12 + +## 0.2.16 + +### Patch Changes + +- Updated dependencies [9a104b9] + - @electric-sql/pglite@0.3.11 + +## 0.2.15 + +### Patch Changes + +- Updated dependencies [ad765ed] + - @electric-sql/pglite@0.3.10 + +## 0.2.14 + +### Patch Changes + +- e40ccad: Upgrade emsdk +- Updated dependencies [e40ccad] + - @electric-sql/pglite@0.3.9 + +## 0.2.13 + +### Patch Changes + +- be677b4: fix pg_dump on Windows systems + + When calling **pg_dump** on Windows system the function fails with an error as the one bellow. + ❗ Notice the double drive letter + `Error: ENOENT: no such file or directory, open 'E:\C:\Users\\AppData\Local\npm-cache\_npx\ba4f1959e38407b5\node_modules\@electric-sql\pglite-tools\dist\pg_dump.wasm'` + + The problem is in execPgDump function at line + `const blob = await fs.readFile(bin.toString().slice(7))` + I think the intention here was to remove `file://` from the begging of the path. However this is not necesarry readFile can handle URL objects. + Moreover this will fail on Windows becase the slice creates a path like '/C:/...' and the readFile function will add the extra drive letter + +- Updated dependencies [f12a582] +- Updated dependencies [bd263aa] + - @electric-sql/pglite@0.3.8 + +## 0.2.12 + +### Patch Changes + +- Updated dependencies [0936962] + - @electric-sql/pglite@0.3.7 + +## 0.2.11 + +### Patch Changes + +- Updated dependencies [6898469] +- Updated dependencies [469be18] +- Updated dependencies [64e33c7] + - @electric-sql/pglite@0.3.6 + +## 0.2.10 + +### Patch Changes + +- 8172b72: new pg_dump wasm blob +- Updated dependencies [6653899] +- Updated dependencies [5f007fc] + - @electric-sql/pglite@0.3.5 + +## 0.2.9 + +### Patch Changes + +- 38a55d0: fix cjs/esm misconfigurations +- Updated dependencies [1fcaa3e] +- Updated dependencies [38a55d0] +- Updated dependencies [aac7003] +- Updated dependencies [8ca254d] + - @electric-sql/pglite@0.3.4 + +## 0.2.8 + +### Patch Changes + +- Updated dependencies [ea2c7c7] + - @electric-sql/pglite@0.3.3 + +## 0.2.7 + +### Patch Changes + +- Updated dependencies [e2c654b] + - @electric-sql/pglite@0.3.2 + +## 0.2.6 + +### Patch Changes + +- Updated dependencies [713364e] + - @electric-sql/pglite@0.3.1 + +## 0.2.5 + +### Patch Changes + +- 317fd36: Specify a peer dependency range on @electric-sql/pglite +- Updated dependencies [97e52f7] +- Updated dependencies [4356024] +- Updated dependencies [0033bc7] + - @electric-sql/pglite@0.3.0 + +## 0.2.4 + +### Patch Changes + +- bbfa9f1: Restore SEARCH_PATH after pg_dump + +## 0.2.3 + +### Patch Changes + +- 8545760: pg_dump error messages set on the thrown Error +- d26e658: Run a DEALLOCATE ALL after each pg_dump to cleanup the prepared statements. + +## 0.2.2 + +### Patch Changes + +- 17c9875: add node imports to the package.json browser excludes + +## 0.2.1 + +### Patch Changes + +- 6547374: Alpha version of pg_dump support in the browser and Node using a WASM build of pg_dump diff --git a/packages/pglite-initdb/README.md b/packages/pglite-initdb/README.md new file mode 100644 index 000000000..da3c2dac7 --- /dev/null +++ b/packages/pglite-initdb/README.md @@ -0,0 +1,72 @@ +# pglite-tools + +A selection of tools for working with [PGlite](https://github.com/electric-sql/pglite) databases, including pg_dump. + +Install with: + +```bash +npm install @electric-sql/pglite-tools +``` + +## `pgDump` + +pg_dump is a tool for dumping a PGlite database to a SQL file, this is a WASM build of pg_dump that can be used in a browser or other JavaScript environments. You can read more about pg_dump [in the Postgres docs](https://www.postgresql.org/docs/current/app-pgdump.html). + +Note: pg_dump will execute `DEALLOCATE ALL;` after each dump. Since this is running on the same (single) connection, any prepared statements that you have made before running pg_dump will be affected. + +### Options + +- `pg`: A PGlite instance. +- `args`: An array of arguments to pass to pg_dump - see [pg_dump docs](https://www.postgresql.org/docs/current/app-pgdump.html) for more details. +- `fileName`: The name of the file to write the dump to, defaults to `dump.sql`. + +There are a number of arguments that are automatically added to the end of the command, these are: + +- `--inserts` - use inserts format for the output, this ensures that the dump can be restored by simply passing the output to `pg.exec()`. +- `-j 1` - concurrency level, set to 1 as multithreading isn't supported. +- `-f /tmp/out.sql` - the output file is always written to `/tmp/out.sql` in the virtual file system. +- `-U postgres` - use the postgres user is hard coded. + +### Returns + +- A `File` object containing the dump. + +### Caveats + +- After restoring a dump, you might want to set the same search path as the initial db. + +### Example + +```typescript +import { PGlite } from '@electric-sql/pglite' +import { pgDump } from '@electric-sql/pglite-tools/pg_dump' + +const pg = await PGlite.create() + +// Create a table and insert some data +await pg.exec(` + CREATE TABLE test ( + id SERIAL PRIMARY KEY, + name TEXT + ); +`) +await pg.exec(` + INSERT INTO test (name) VALUES ('test'); +`) + +// store the current search path so it can be used in the restored db +const initialSearchPath = (await pg1.query<{ search_path: string }>('SHOW SEARCH_PATH;')).rows[0].search_path + +// Dump the database to a file +const dump = await pgDump({ pg }) +// Get the dump text - used for restore +const dumpContent = await dump.text() + +// Create a new database +const restoredPG = await PGlite.create() +// ... and restore it using the dump +await restoredPG.exec(dumpContent) + +// optional - after importing, set search path back to the initial one +await restoredPG.exec(`SET search_path TO ${initialSearchPath};`); +``` diff --git a/packages/pglite-initdb/eslint.config.js b/packages/pglite-initdb/eslint.config.js new file mode 100644 index 000000000..d85bdb2aa --- /dev/null +++ b/packages/pglite-initdb/eslint.config.js @@ -0,0 +1,29 @@ +import globals from 'globals' +import rootConfig from '../../eslint.config.js' + +export default [ + ...rootConfig, + { + ignores: ['release/**/*', 'examples/**/*', 'dist/**/*'], + }, + { + languageOptions: { + globals: { + ...globals.browser, + ...globals.node, + }, + }, + rules: { + ...rootConfig.rules, + '@typescript-eslint/no-explicit-any': 'off', + }, + }, + { + files: ['tests/targets/deno/**/*.js'], + languageOptions: { + globals: { + Deno: false, + }, + }, + }, +] diff --git a/packages/pglite-initdb/package.json b/packages/pglite-initdb/package.json new file mode 100644 index 000000000..901594f18 --- /dev/null +++ b/packages/pglite-initdb/package.json @@ -0,0 +1,67 @@ +{ + "name": "@electric-sql/pglite-initdb", + "version": "0.0.1", + "description": "initdb as wasm", + "author": "Electric DB Limited", + "homepage": "https://pglite.dev", + "license": "Apache-2.0", + "keywords": [ + "postgres", + "sql", + "database", + "wasm", + "pglite", + "initdb" + ], + "private": false, + "publishConfig": { + "access": "public" + }, + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "./initdb": { + "import": { + "types": "./dist/initdb.d.ts", + "default": "./dist/initdb.js" + }, + "require": { + "types": "./dist/initdb.d.cts", + "default": "./dist/initdb.cjs" + } + } + }, + "scripts": { + "build": "tsup", + "check:exports": "attw . --pack --profile node16", + "lint": "eslint ./src ./tests --report-unused-disable-directives --max-warnings 0", + "format": "prettier --write ./src ./tests", + "typecheck": "tsc", + "stylecheck": "pnpm lint && prettier --check ./src ./tests", + "test": "vitest", + "prepublishOnly": "pnpm check:exports" + }, + "browser": { + "fs": false, + "fs/promises": false + }, + "devDependencies": { + "@arethetypeswrong/cli": "^0.18.1", + "@types/emscripten": "^1.41.1", + "@types/node": "^20.16.11", + "tsx": "^4.19.2", + "vitest": "^1.3.1" + } +} diff --git a/packages/pglite-initdb/src/argsParser.js b/packages/pglite-initdb/src/argsParser.js new file mode 100644 index 000000000..650ddfe0c --- /dev/null +++ b/packages/pglite-initdb/src/argsParser.js @@ -0,0 +1,238 @@ +'use strict' + +// '<(' is process substitution operator and +// can be parsed the same as control operator +var CONTROL = + '(?:' + + [ + '\\|\\|', + '\\&\\&', + ';;', + '\\|\\&', + '\\<\\(', + '\\<\\<\\<', + '>>', + '>\\&', + '<\\&', + '[&;()|<>]', + ].join('|') + + ')' +var controlRE = new RegExp('^' + CONTROL + '$') +var META = '|&;()<> \\t' +var SINGLE_QUOTE = '"((\\\\"|[^"])*?)"' +var DOUBLE_QUOTE = "'((\\\\'|[^'])*?)'" +var hash = /^#$/ + +var SQ = "'" +var DQ = '"' +var DS = '$' + +var TOKEN = '' +var mult = 0x100000000 // Math.pow(16, 8); +for (var i = 0; i < 4; i++) { + TOKEN += (mult * Math.random()).toString(16) +} +var startsWithToken = new RegExp('^' + TOKEN) + +function matchAll(s, r) { + var origIndex = r.lastIndex + + var matches = [] + var matchObj + + while ((matchObj = r.exec(s))) { + matches.push(matchObj) + if (r.lastIndex === matchObj.index) { + r.lastIndex += 1 + } + } + + r.lastIndex = origIndex + + return matches +} + +function getVar(env, pre, key) { + var r = typeof env === 'function' ? env(key) : env[key] + if (typeof r === 'undefined' && key != '') { + r = '' + } else if (typeof r === 'undefined') { + r = '$' + } + + if (typeof r === 'object') { + return pre + TOKEN + JSON.stringify(r) + TOKEN + } + return pre + r +} + +function parseInternal(string, env, opts) { + if (!opts) { + opts = {} + } + var BS = opts.escape || '\\' + var BAREWORD = '(\\' + BS + '[\'"' + META + ']|[^\\s\'"' + META + '])+' + + var chunker = new RegExp( + [ + '(' + CONTROL + ')', // control chars + '(' + BAREWORD + '|' + SINGLE_QUOTE + '|' + DOUBLE_QUOTE + ')+', + ].join('|'), + 'g', + ) + + var matches = matchAll(string, chunker) + + if (matches.length === 0) { + return [] + } + if (!env) { + env = {} + } + + var commented = false + + return matches + .map(function (match) { + var s = match[0] + if (!s || commented) { + return void undefined + } + if (controlRE.test(s)) { + return { op: s } + } + + // Hand-written scanner/parser for Bash quoting rules: + // + // 1. inside single quotes, all characters are printed literally. + // 2. inside double quotes, all characters are printed literally + // except variables prefixed by '$' and backslashes followed by + // either a double quote or another backslash. + // 3. outside of any quotes, backslashes are treated as escape + // characters and not printed (unless they are themselves escaped) + // 4. quote context can switch mid-token if there is no whitespace + // between the two quote contexts (e.g. all'one'"token" parses as + // "allonetoken") + var quote = false + var esc = false + var out = '' + var isGlob = false + var i + + function parseEnvVar() { + i += 1 + var varend + var varname + var char = s.charAt(i) + + if (char === '{') { + i += 1 + if (s.charAt(i) === '}') { + throw new Error('Bad substitution: ' + s.slice(i - 2, i + 1)) + } + varend = s.indexOf('}', i) + if (varend < 0) { + throw new Error('Bad substitution: ' + s.slice(i)) + } + varname = s.slice(i, varend) + i = varend + } else if (/[*@#?$!_-]/.test(char)) { + varname = char + i += 1 + } else { + var slicedFromI = s.slice(i) + varend = slicedFromI.match(/[^\w\d_]/) + if (!varend) { + varname = slicedFromI + i = s.length + } else { + varname = slicedFromI.slice(0, varend.index) + i += varend.index - 1 + } + } + return getVar(env, '', varname) + } + + for (i = 0; i < s.length; i++) { + var c = s.charAt(i) + isGlob = isGlob || (!quote && (c === '*' || c === '?')) + if (esc) { + out += c + esc = false + } else if (quote) { + if (c === quote) { + quote = false + } else if (quote == SQ) { + out += c + } else { + // Double quote + if (c === BS) { + i += 1 + c = s.charAt(i) + if (c === DQ || c === BS || c === DS) { + out += c + } else { + out += BS + c + } + } else if (c === DS) { + out += parseEnvVar() + } else { + out += c + } + } + } else if (c === DQ || c === SQ) { + quote = c + } else if (controlRE.test(c)) { + return { op: s } + } else if (hash.test(c)) { + commented = true + var commentObj = { comment: string.slice(match.index + i + 1) } + if (out.length) { + return [out, commentObj] + } + return [commentObj] + } else if (c === BS) { + esc = true + } else if (c === DS) { + out += parseEnvVar() + } else { + out += c + } + } + + if (isGlob) { + return { op: 'glob', pattern: out } + } + + return out + }) + .reduce(function (prev, arg) { + // finalize parsed arguments + // TODO: replace this whole reduce with a concat + return typeof arg === 'undefined' ? prev : prev.concat(arg) + }, []) +} + +export default function parse(s, env, opts) { + var mapped = parseInternal(s, env, opts) + if (typeof env !== 'function') { + return mapped + } + return mapped.reduce(function (acc, s) { + if (typeof s === 'object') { + return acc.concat(s) + } + var xs = s.split(RegExp('(' + TOKEN + '.*?' + TOKEN + ')', 'g')) + if (xs.length === 1) { + return acc.concat(xs[0]) + } + return acc.concat( + xs.filter(Boolean).map(function (x) { + if (startsWithToken.test(x)) { + return JSON.parse(x.split(TOKEN)[1]) + } + return x + }), + ) + }, []) +} diff --git a/packages/pglite-initdb/src/index.ts b/packages/pglite-initdb/src/index.ts new file mode 100644 index 000000000..7970301e5 --- /dev/null +++ b/packages/pglite-initdb/src/index.ts @@ -0,0 +1 @@ +export * from './initdb' diff --git a/packages/pglite-initdb/src/initdb.ts b/packages/pglite-initdb/src/initdb.ts new file mode 100644 index 000000000..96c58ec3f --- /dev/null +++ b/packages/pglite-initdb/src/initdb.ts @@ -0,0 +1,246 @@ +import InitdbModFactory, { InitdbMod } from './initdbModFactory' +import parse from './argsParser' + +function assert(condition: unknown, message?: string): asserts condition { + if (!condition) { + throw new Error(message ?? 'Assertion failed') + } +} +// import fs from 'node:fs' + +export const PGDATA = '/pglite/data' + +const initdbExePath = '/pglite/bin/initdb' +const pgstdoutPath = '/pglite/pgstdout' +const pgstdinPath = '/pglite/pgstdin' + +/** + * Interface defining what initdb needs from a PGlite instance. + * This avoids a circular dependency between pglite and pglite-initdb. + */ +export interface PGliteForInitdb { + Module: { + HEAPU8: Uint8Array + stringToUTF8OnStack(str: string): number + _pgl_freopen(path: number, mode: number, fd: number): void + FS: any + } + callMain(args: string[]): number +} + +interface ExecResult { + exitCode: number + stderr: string + stdout: string + dataFolder: string +} + +function log(debug?: number, ...args: any[]) { + if (debug && debug > 0) { + console.log('initdb: ', ...args) + } +} + +async function execInitdb({ + pg, + debug, + args, +}: { + pg: PGliteForInitdb + debug?: number + args: string[] +}): Promise { + let system_fn, popen_fn, pclose_fn + + let needToCallPGmain = false + let postgresArgs: string[] = [] + + let pgMainResult = 0 + + // let pglite_stdin_fd = -1 + let initdb_stdin_fd = -1 + // let pglite_stdout_fd = -1 + let initdb_stdout_fd = -1 + // let i_pgstdin = 0 + let stderrOutput: string = '' + let stdoutOutput: string = '' + + const callPgMain = (args: string[]) => { + const firstArg = args.shift() + log(debug, 'initdb: firstArg', firstArg) + assert(firstArg === '/pglite/bin/postgres', `trying to execute ${firstArg}`) + + pg.Module.HEAPU8.set(origHEAPU8) + + log(debug, 'executing pg main with', args) + const result = pg.callMain(args) + + log(debug, result) + + postgresArgs = [] + + return result + } + + const origHEAPU8 = pg.Module.HEAPU8.slice() + + const emscriptenOpts: Partial = { + arguments: args, + noExitRuntime: false, + thisProgram: initdbExePath, + // Provide a stdin that returns EOF to avoid browser prompt + stdin: () => null, + print: (text) => { + stdoutOutput += text + log(debug, 'initdbout', text) + }, + printErr: (text) => { + stderrOutput += text + log(debug, 'initdberr', text) + }, + preRun: [ + // (mod: InitdbMod) => { + // mod.FS.init(initdb_stdin, initdb_stdout, null) + // }, + (mod: InitdbMod) => { + mod.onRuntimeInitialized = () => { + // default $HOME in emscripten is /home/web_user + system_fn = mod.addFunction((cmd_ptr: number) => { + postgresArgs = getArgs(mod.UTF8ToString(cmd_ptr)) + return callPgMain(postgresArgs) + }, 'pi') + + mod._pgl_set_system_fn(system_fn) + + popen_fn = mod.addFunction((cmd_ptr: number, mode: number) => { + const smode = mod.UTF8ToString(mode) + postgresArgs = getArgs(mod.UTF8ToString(cmd_ptr)) + + if (smode === 'r') { + pgMainResult = callPgMain(postgresArgs) + return initdb_stdin_fd + } else { + if (smode === 'w') { + needToCallPGmain = true + return initdb_stdout_fd + } else { + throw `Unexpected popen mode value ${smode}` + } + } + }, 'ppi') + + mod._pgl_set_popen_fn(popen_fn) + + pclose_fn = mod.addFunction((stream: number) => { + if (stream === initdb_stdin_fd || stream === initdb_stdout_fd) { + // if the last popen had mode w, execute now postgres' main() + if (needToCallPGmain) { + needToCallPGmain = false + pgMainResult = callPgMain(postgresArgs) + } + // const closeResult = mod._fclose(stream) + // console.log(closeResult) + return pgMainResult + } else { + return mod._pclose(stream) + } + }, 'pi') + + mod._pgl_set_pclose_fn(pclose_fn) + + { + const pglite_stdin_path = pg.Module.stringToUTF8OnStack(pgstdinPath) + const rmode = pg.Module.stringToUTF8OnStack('r') + pg.Module._pgl_freopen(pglite_stdin_path, rmode, 0) + const pglite_stdout_path = + pg.Module.stringToUTF8OnStack(pgstdoutPath) + const wmode = pg.Module.stringToUTF8OnStack('w') + pg.Module._pgl_freopen(pglite_stdout_path, wmode, 1) + } + + { + const initdb_path = mod.stringToUTF8OnStack(pgstdoutPath) + const rmode = mod.stringToUTF8OnStack('r') + initdb_stdin_fd = mod._fopen(initdb_path, rmode) + + const path = mod.stringToUTF8OnStack(pgstdinPath) + const wmode = mod.stringToUTF8OnStack('w') + initdb_stdout_fd = mod._fopen(path, wmode) + } + + // pg.Module.FS.chdir(PGDATA) + } + }, + (mod: InitdbMod) => { + mod.ENV.PGDATA = PGDATA + }, + (mod: InitdbMod) => { + mod.FS.mkdir('/pglite') + mod.FS.mount( + mod.PROXYFS, + { + root: '/pglite', + fs: pg.Module.FS, + }, + '/pglite', + ) + }, + ], + } + + const initDbMod = await InitdbModFactory(emscriptenOpts) + + log(debug, 'calling initdb.main with', args) + const result = initDbMod.callMain(args) + + // pg.Module.HEAPU8.set(origHEAPU8) + + return { + exitCode: result, + stderr: stderrOutput, + stdout: stdoutOutput, + dataFolder: PGDATA, + } +} + +interface InitdbOptions { + pg: PGliteForInitdb + debug?: number + args?: string[] +} + +function getArgs(cmd: string) { + const a: string[] = [] + const parsed = parse(cmd) + // console.log("parsed args", parsed) + for (let i = 0; i < parsed.length; i++) { + if (parsed[i].op) break + a.push(parsed[i]) + } + return a +} + +/** + * Execute initdb + */ +export async function initdb({ + pg, + debug, + args, +}: InitdbOptions): Promise { + const execResult = await execInitdb({ + pg, + debug, + args: [ + '--allow-group-access', + '--encoding', + 'UTF8', + '--locale=C.UTF-8', + '--locale-provider=libc', + '--auth=trust', + ...(args ?? []), + ], + }) + + return execResult +} diff --git a/packages/pglite-initdb/src/initdbModFactory.ts b/packages/pglite-initdb/src/initdbModFactory.ts new file mode 100644 index 000000000..323bf83fe --- /dev/null +++ b/packages/pglite-initdb/src/initdbModFactory.ts @@ -0,0 +1,61 @@ +import InitdbModFactory from '../release/initdb' + +type IDBFS = Emscripten.FileSystemType & { + quit: () => void + dbs: Record +} + +export type FS = typeof FS & { + filesystems: { + MEMFS: Emscripten.FileSystemType + NODEFS: Emscripten.FileSystemType + IDBFS: IDBFS + } + quit: () => void +} + +export interface InitdbMod + extends Omit { + preInit: Array<{ (mod: InitdbMod): void }> + preRun: Array<{ (mod: InitdbMod): void }> + postRun: Array<{ (mod: InitdbMod): void }> + thisProgram: string + stdin: (() => number | null) | null + ENV: Record + FS: FS + PROXYFS: Emscripten.FileSystemType + WASM_PREFIX: string + INITIAL_MEMORY: number + UTF8ToString: (ptr: number, maxBytesToRead?: number) => string + stringToUTF8OnStack: (s: string) => number + ___errno_location: () => number + _strerror: (errno: number) => number + _pgl_set_rw_cbs: (read_cb: number, write_cb: number) => void + _pgl_set_system_fn: (system_fn: number) => void + _pgl_set_popen_fn: (popen_fn: number) => void + _pgl_set_pclose_fn: (pclose_fn: number) => void + _pgl_set_pipe_fn: (pipe_fn: number) => void + _pclose: (stream: number) => number + _pipe: (fd: number) => number + _pgl_freopen: (filepath: number, mode: number, stream: number) => number + // _pgl_set_fgets_fn: (fgets_fn: number) => void + // _pgl_set_fputs_fn: (fputs_fn: number) => void + // _pgl_set_errno: (errno: number) => number + // _fgets: (str: number, size: number, stream: number) => number + // _fputs: (s: number, stream: number) => number + _fopen: (path: number, mode: number) => number + _fclose: (stream: number) => number + _fflush: (stream: number) => number + addFunction: (fn: CallableFunction, signature: string) => number + removeFunction: (f: number) => void + callMain: (args: string[]) => number + onExit: (status: number) => void + print: (test: string) => void + printErr: (text: string) => void +} + +type PgDumpFactory = ( + moduleOverrides?: Partial, +) => Promise + +export default InitdbModFactory as PgDumpFactory diff --git a/packages/pglite-initdb/tests/initdb.test.ts b/packages/pglite-initdb/tests/initdb.test.ts new file mode 100644 index 000000000..b1b19c6af --- /dev/null +++ b/packages/pglite-initdb/tests/initdb.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect } from 'vitest' +import { PGlite } from '@electric-sql/pglite' +import { initdb } from '../dist/initdb.js' + +describe('initdb', () => { + it('should init a database', async () => { + // const pg = await PGlite.create() + const result = await initdb({ args: ['--no-clean'] }) + expect(result.exitCode).toBe(0) + expect(result.stdout).contains( + 'You can now start the database server using', + ) + }) + it('should init a database and exec a simple query', async () => { + const pg = await PGlite.create() + const result = await initdb({ pg, args: ['--no-clean'], debug: 5 }) + expect(result.exitCode).toBe(0) + expect(result.stdout).contains( + 'You can now start the database server using', + ) + const selectResult = await pg.exec('SELECT 1') + console.log(selectResult) + }) + + it('should init a database and run simple query', async () => { + const pg = await PGlite.create() + const result = await initdb({ pg, args: ['--no-clean'], debug: 5 }) + expect(result.exitCode).toBe(0) + expect(result.stdout).contains( + 'You can now start the database server using', + ) + pg.startInSingle() + const selectResult = await pg.query('SELECT 1;') + console.log(selectResult) + }) + + it('should init a database and create a table query', async () => { + const pg = await PGlite.create() + const result = await initdb({ pg, args: ['--no-clean'], debug: 5 }) + expect(result.exitCode).toBe(0) + expect(result.stdout).contains( + 'You can now start the database server using', + ) + pg.startInSingle() + await pg.query(`CREATE TABLE IF NOT EXISTS test ( + id SERIAL PRIMARY KEY, + name TEXT); + `) + + const multiStatementResult = await pg.exec(` + INSERT INTO test (name) VALUES ('test'); + UPDATE test SET name = 'test2'; + SELECT * FROM test; + `) + + expect(multiStatementResult).toEqual([ + { + affectedRows: 1, + rows: [], + fields: [], + }, + { + affectedRows: 2, + rows: [], + fields: [], + }, + { + rows: [{ id: 1, name: 'test2' }], + fields: [ + { name: 'id', dataTypeID: 23 }, + { name: 'name', dataTypeID: 25 }, + ], + affectedRows: 2, + }, + ]) + + await pg.close() + // console.log(selectResult) + }) +}) diff --git a/packages/pglite-initdb/tests/setup.ts b/packages/pglite-initdb/tests/setup.ts new file mode 100644 index 000000000..2ac9bc141 --- /dev/null +++ b/packages/pglite-initdb/tests/setup.ts @@ -0,0 +1,15 @@ +import { beforeAll } from 'vitest' +import { execSync } from 'child_process' +import { existsSync } from 'fs' +import { join } from 'path' + +beforeAll(() => { + // Check if we need to build + const distPath = join(__dirname, '../dist') + const wasmPath = join(distPath, 'pg_dump.wasm') + + if (!existsSync(wasmPath)) { + console.log('Building project before running tests...') + execSync('pnpm build', { stdio: 'inherit' }) + } +}) diff --git a/packages/pglite-initdb/tsconfig.json b/packages/pglite-initdb/tsconfig.json new file mode 100644 index 000000000..ac9f11d02 --- /dev/null +++ b/packages/pglite-initdb/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": [ + "@types/emscripten", + "node" + ] + }, + "include": ["src", "tsup.config.ts", "vitest.config.ts"] +} diff --git a/packages/pglite-initdb/tsup.config.ts b/packages/pglite-initdb/tsup.config.ts new file mode 100644 index 000000000..0c139e4f5 --- /dev/null +++ b/packages/pglite-initdb/tsup.config.ts @@ -0,0 +1,28 @@ +import { cpSync } from 'fs' +import { resolve } from 'path' +import { defineConfig } from 'tsup' + +const entryPoints = [ + 'src/index.ts', + 'src/initdb.ts', +] + +const minify = process.env.DEBUG === 'true' ? false : true + +export default defineConfig([ + { + entry: entryPoints, + sourcemap: true, + dts: { + entry: entryPoints, + resolve: true, + }, + clean: true, + minify: minify, + shims: true, + format: ['esm', 'cjs'], + onSuccess: async () => { + cpSync(resolve('release/initdb.wasm'), resolve('dist/initdb.wasm')) + } + }, +]) diff --git a/packages/pglite-initdb/vitest.config.ts b/packages/pglite-initdb/vitest.config.ts new file mode 100644 index 000000000..c2c0f4b1e --- /dev/null +++ b/packages/pglite-initdb/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + testTimeout: 30000, + setupFiles: ['./tests/setup.ts'], + }, +}) diff --git a/packages/pglite/package.json b/packages/pglite/package.json index 770ba869c..2b5b2f34a 100644 --- a/packages/pglite/package.json +++ b/packages/pglite/package.json @@ -212,6 +212,9 @@ "stylecheck": "pnpm lint && prettier --check ./src ./tests", "prepublishOnly": "pnpm check:exports" }, + "dependencies": { + "@electric-sql/pglite-initdb": "workspace:*" + }, "devDependencies": { "@arethetypeswrong/cli": "^0.18.1", "@electric-sql/pg-protocol": "workspace:*", diff --git a/packages/pglite/scripts/bundle-wasm.ts b/packages/pglite/scripts/bundle-wasm.ts index 73c2b32a9..15f24039a 100644 --- a/packages/pglite/scripts/bundle-wasm.ts +++ b/packages/pglite/scripts/bundle-wasm.ts @@ -52,6 +52,9 @@ const copyFiles = async (srcDir: string, destDir: string) => { async function main() { await copyFiles('./release', './dist') + // Copy initdb.wasm from pglite-initdb package + await fs.copyFile('../pglite-initdb/release/initdb.wasm', './dist/initdb.wasm') + console.log('Copied initdb.wasm to ./dist/initdb.wasm') await findAndReplaceInDir('./dist', /\.\.\/release\//g, './', ['.js', '.cjs']) await findAndReplaceInDir('./dist/contrib', /\.\.\/release\//g, '', [ '.js', diff --git a/packages/pglite/src/base.ts b/packages/pglite/src/base.ts index e18efb911..6f23754c6 100644 --- a/packages/pglite/src/base.ts +++ b/packages/pglite/src/base.ts @@ -313,6 +313,12 @@ export abstract class BasePGlite } throw e } finally { + results.push( + ...(await this.#execProtocolNoSync( + serializeProtocol.flush(), + options, + )), + ) results.push( ...(await this.#execProtocolNoSync( serializeProtocol.sync(), diff --git a/packages/pglite/src/extensionUtils.ts b/packages/pglite/src/extensionUtils.ts index 6b00baaab..c70bd6012 100644 --- a/packages/pglite/src/extensionUtils.ts +++ b/packages/pglite/src/extensionUtils.ts @@ -149,6 +149,74 @@ function copyToFS(filePath: string, mod: PostgresMod, entry: any) { } } +const preloadedExtensions = [ + '.so', + '.jpg', + '.jpeg', + '.png', + '.bmp', + '.ogg', + '.wav', + '.mp3', +] + +function canPreloadFile(fileName: string) { + return preloadedExtensions.some((ext) => fileName.endsWith(ext)) +} + +export function loadFiles( + mod: PostgresMod, + files: [ + { + fullPath: string + bytes: Uint8Array + }, + ], + log: (...args: any[]) => void, +) { + files.forEach((f) => { + loadFile(mod, f.fullPath, f.bytes, log) + }) +} + +export function loadFile( + mod: PostgresMod, + fullPath: string, + bytes: Uint8Array, + log: (...args: any[]) => void, +) { + const fileName = fullPath.split('/').pop()! + const dirPath = dirname(fullPath) + + if (canPreloadFile(fileName)) { + const extOk = (...args: any[]) => { + log('pgfs:ext OK', fullPath, args) + } + const extFail = (...args: any[]) => { + log('pgfs:ext FAIL', fullPath, args) + } + mod.FS.createPreloadedFile( + dirPath, + fileName!.slice(0, -3), + bytes as any, // There is a type error in Emscripten's FS.createPreloadedFile, this excepts a Uint8Array, but the type is defined as any + true, + true, + extOk, + extFail, + false, + ) + } else { + try { + if (mod.FS.analyzePath(dirPath).exists === false) { + mod.FS.mkdirTree(dirPath) + } + mod.FS.writeFile(fullPath, bytes) + } catch (e) { + console.error(`Error writing file ${fullPath}`, e) + } + } +} + function dirname(path: string) { const last = path.lastIndexOf('/') if (last > 0) { diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index a8f8ba06f..f976a55bd 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -1,6 +1,10 @@ import { Mutex } from 'async-mutex' import { BasePGlite } from './base.js' -import { loadExtensionBundle, loadExtensions } from './extensionUtils.js' +import { + loadExtensionBundle, + loadExtensions, + loadFiles, +} from './extensionUtils.js' import { type Filesystem, loadFs, diff --git a/packages/pglite/tests/contrib/amcheck.test.js b/packages/pglite/tests/contrib/amcheck.test.js index 87ec07efc..4307f449c 100644 --- a/packages/pglite/tests/contrib/amcheck.test.js +++ b/packages/pglite/tests/contrib/amcheck.test.js @@ -38,7 +38,7 @@ it('amcheck', async () => { { bt_index_check: '', relname: 'pg_description_o_c_o_index', - relpages: 23, + relpages: 27, }, { bt_index_check: '', @@ -65,6 +65,11 @@ it('amcheck', async () => { relname: 'pg_depend_reference_index', relpages: 8, }, + { + bt_index_check: '', + relname: 'pg_collation_name_enc_nsp_index', + relpages: 7, + }, { bt_index_check: '', relname: 'pg_amop_fam_strat_index', diff --git a/packages/pglite/tests/contrib/file_fdw.test.ts b/packages/pglite/tests/contrib/file_fdw.test.ts new file mode 100644 index 000000000..3ea6f1f16 --- /dev/null +++ b/packages/pglite/tests/contrib/file_fdw.test.ts @@ -0,0 +1,55 @@ +import { it, expect } from 'vitest' +import { PGlite } from '../../dist/index.js' +import { file_fdw } from '../../dist/contrib/file_fdw.js' + +it('file_fdw', async () => { + const pg = await PGlite.create({ + extensions: { + file_fdw, + }, + }) + + await pg.exec('CREATE EXTENSION IF NOT EXISTS file_fdw;') + await pg.exec('CREATE SERVER file_server FOREIGN DATA WRAPPER file_fdw;') + await pg.exec(`CREATE FOREIGN TABLE file_contents (line text) + SERVER file_server + OPTIONS ( + filename '/pglite/bin/postgres', + format 'text' + );`) + + const contents = await pg.query(`SELECT * FROM file_contents;`) + expect(contents.rows).toEqual([ + { + line: 'PGlite is the best!', + }, + ]) +}) + +it('file_fdw with loadFile', async () => { + const pg = await PGlite.create({ + extensions: { + file_fdw, + }, + }) + + const text = 'PGlite %^&!@#' + const data = new TextEncoder().encode(text) + pg.loadFiles([{ fullPath: '/tmp/dummy1/dummy2/myfile.txt', bytes: data }]) + + await pg.exec('CREATE EXTENSION IF NOT EXISTS file_fdw;') + await pg.exec('CREATE SERVER file_server FOREIGN DATA WRAPPER file_fdw;') + await pg.exec(`CREATE FOREIGN TABLE file_contents (line text) + SERVER file_server + OPTIONS ( + filename '/tmp/dummy1/dummy2/myfile.txt', + format 'text' + );`) + + const contents = await pg.query(`SELECT * FROM file_contents;`) + expect(contents.rows).toEqual([ + { + line: text, + }, + ]) +}) diff --git a/packages/pglite/tsup.config.ts b/packages/pglite/tsup.config.ts index 5e3de0639..75f34ef49 100644 --- a/packages/pglite/tsup.config.ts +++ b/packages/pglite/tsup.config.ts @@ -53,6 +53,7 @@ export default defineConfig([ }, clean: true, external: ['../release/pglite.js', '../release/pglite.cjs'], + noExternal: ['@electric-sql/pglite-initdb'], esbuildPlugins: [replaceAssertPlugin], minify: minify, shims: true, // Convert import.meta.url to a shim for CJS diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ba61a8abc..26948de66 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -161,6 +161,10 @@ importers: version: 2.1.2(@types/node@20.16.11)(jsdom@24.1.3)(terser@5.34.1) packages/pglite: + dependencies: + '@electric-sql/pglite-initdb': + specifier: workspace:* + version: link:../pglite-initdb devDependencies: '@arethetypeswrong/cli': specifier: ^0.18.1 diff --git a/postgres-pglite b/postgres-pglite index e01963726..7aa98cfdd 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit e01963726df03e4700de48b69d1ac16ea5e20bef +Subproject commit 7aa98cfdddfa8d5ec660ad2295bc8fd187ef08bb