ICU support (new)

.github/workflows/build_and_test.yml

@@ -54,12 +54,31 @@ jobs:
         working-directory: ${{ github.workspace }}
         run: pnpm install
 
-      - name: Build all
+      - name: Get postgres-pglite submodule hash
+        id: postgres-pglite-commit-id
+        run: echo "hash=$(git rev-parse HEAD:postgres-pglite)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache WASM build artifacts
+        id: postgres-pglite-cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            packages/pglite/release
+            packages/pglite-tools/release
+            packages/pglite-postgis/release
+          key: wasm-build-${{ steps.postgres-pglite-commit-id.outputs.hash }}
+
+      - name: Build all if necessary
+        if: steps.postgres-pglite-cache.outputs.cache-hit != 'true'
         working-directory: ${{ github.workspace }}
         env:
           PGSRC: ${{ github.workspace }}/postgres-pglite
-        run: |
-          pnpm build:all
+        run: pnpm build:all
+
+      - name: Build only typeScript packages if postgres-pglite artifacts already available
+        if: steps.postgres-pglite-cache.outputs.cache-hit == 'true'
+        working-directory: ${{ github.workspace }}
+        run: pnpm ts:build
 
       - name: Upload PGlite Interim to Github artifacts
         id: upload-pglite-interim-build-files
@@ -156,7 +175,7 @@ jobs:
       - name: Test Deno
         run: pnpm test:deno
 
-      - name: Pack for distribution
+      - name: Pack pglite for distribution
         run: pnpm pack
 
       - name: Upload PGlite distribution artifact
@@ -194,6 +213,17 @@ jobs:
             - PGlite with node v${{ matrix.node }}: ${{ steps.upload-pglite-package.outputs.artifact-url }}
           edit-mode: append
 
+      - name: Pack pglite-icu-full for distribution
+        working-directory: ./packages/pglite-icu-full
+        run: pnpm build && pnpm pack
+
+      - name: Upload pglite-icu-full distribution artifact
+        id: upload-pglite-icu-full
+        uses: actions/upload-artifact@v4
+        with:
+          name: pglite-icu-full-node-v${{ matrix.node }}
+          path: ./packages/pglite-icu-full/electric-sql-pglite-icu-full-*.tgz          
+
   build-and-test-pglite-dependents:
     name: Build and Test packages dependent on PGlite
     runs-on: blacksmith-32vcpu-ubuntu-2204
@@ -258,7 +288,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: pglite-postgis-dist-node-v${{ matrix.node }}
-          path: ./packages/pglite-postgis/dist/*                  
+          path: ./packages/pglite-postgis/dist/*         
 
   publish-website-with-demos:
     name: Publish website with demos
@@ -303,7 +333,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: pglite-postgis-dist-node-v20.x
-          path: ./packages/pglite-postgis/dist/
+          path: ./packages/pglite-postgis/dist/ 
 
       - name: Install dependencies
         run: pnpm install --frozen-lockfile

packages/pglite-icu-full/.gitignore

@@ -0,0 +1 @@
+release/

packages/pglite-icu-full/README.md

@@ -0,0 +1,35 @@
+# pglite-icu-full
+
+A package containing all the resources from [libicu](https://github.com/unicode-org/icu) that can be used with PGlite to build localized applications.
+
+## Installation
+
+```bash
+npm install @electric-sql/pglite-icu-full
+# or
+yarn add @electric-sql/pglite-icu-full
+# or
+pnpm add @electric-sql/pglite-icu-full
+```
+
+## Usage
+
+This loads the entire locale set provided by libicu, which might be quite large.
+
+```typescript
+import { PGlite } from '@electric-sql/pglite'
+import { icuDataDir } from '@electric-sql/pglite-icu-full'
+
+// Create a PGlite instance with the icu resources
+const pg = await PGlite.create({
+  icuDataDir: await icuDataDir(),
+})
+
+// just an example, query the available collations
+const collations = await pg.exec('select * from pg_collation')
+
+```
+
+# Documentation
+
+https://www.postgresql.org/docs/current/locale.html

packages/pglite-icu-full/eslint.config.js

@@ -0,0 +1,29 @@
+import globals from 'globals'
+import rootConfig from '../../eslint.config.js'
+
+export default [
+  ...rootConfig,
+  {
+    ignores: ['release/**/*', 'examples/**/*', 'dist/**/*'],
+  },
+  {
+    languageOptions: {
+      globals: {
+        ...globals.browser,
+        ...globals.node,
+      },
+    },
+    rules: {
+      ...rootConfig.rules,
+      '@typescript-eslint/no-explicit-any': 'off',
+    },
+  },
+  {
+    files: ['tests/targets/deno/**/*.js'],
+    languageOptions: {
+      globals: {
+        Deno: false,
+      },
+    },
+  },
+]

packages/pglite-icu-full/examples/README.md

@@ -0,0 +1,49 @@
+# Generating an ICU package for PGlite
+
+This document shows you how to generate your own icu file that contains only the locales that you want in your PGlite enabled application.
+
+## Download libicu code and data
+
+Currently PGlite is tested to work with libicu v76.1. Get the source and data for it:
+
+wget https://github.com/unicode-org/icu/releases/download/release-76-1/icu4c-76_1-src.tgz
+wget https://github.com/unicode-org/icu/releases/download/release-78.3/icu4c-78.3-data.zip
+
+Important: You must have the data sources in order to use the ICU Data Build Tool. Check for the file icu4c/source/data/locales/root.txt. If that file is missing, you need to download “icu4c-*-data.zip”, delete the old icu4c/source/data directory, and replace it with the data directory from the zip file. If there is a *.dat file in icu4c/source/data/in, that file will be used even if you gave ICU custom filter rules.
+
+## Create a filters.json file
+
+This will allow you to only generate the data that you need.
+
+Here's a simple example:
+```
+{
+  "localeFilter": {
+    "filterType": "locale",
+    "includelist": [
+      "en_US"
+    ]
+  }
+}
+```
+
+For more info, see https://unicode-org.github.io/icu/userguide/icu_data/buildtool.html.
+
+## Build ICU
+
+$ ICU_DATA_FILTER_FILE=<full_path_to_your_filters.json> ./icu/source/configure --with-data-packaging=files --disable-shared --enable-static --disable-tests --disable-samples --disable-extras --disable-icuio --disable-layoutex --prefix=<your_install_dir>
+
+$ make -j && make install
+
+
+## Create an archive with the icu data
+
+The previous steps have installed everything related to ICU in <your_install_dir>. You only need the data files:
+
+$ cd <your_install_dir>/share/icu/76.1/ && tar cvfz icu_76.tgz icudt76l/
+
+Now `icu_76.tgz` contains the localisation data that you can use with PGlite.
+
+## Example
+
+The subfolder `Switzerland` contains the `filter.json` and the generated data file that can be used with PGlite.

packages/pglite-icu-full/examples/Switzerland/filters_switzerland.json

@@ -0,0 +1,29 @@
+{
+  "localeFilter": {
+    "filterType": "locale",
+    "includelist": ["root", "de_CH", "fr_CH", "it_CH", "rm"],
+    "includeChildren": false
+  },
+  "featureFilters": {
+    "brkitr_rules": "exclude",
+    "brkitr_dictionaries": "exclude",
+    "brkitr_tree": "exclude",
+    "conversion_mappings": "exclude",
+    "confusables": "exclude",
+    "curr_supplemental": "exclude",
+    "curr_tree": "exclude",
+    "lang_tree": "exclude",
+    "normalization": "exclude",
+    "region_tree": "exclude",
+    "rbnf_tree": "exclude",
+    "stringprep": "exclude",
+    "zone_tree": "exclude",
+    "translit": "exclude",
+    "unames": "exclude",
+    "ulayout": "exclude",
+    "uemoji": "exclude",
+    "unit_tree": "exclude",
+    "cnvalias": "exclude"
+  },
+  "collationUCAData": "implicithan"
+}

packages/pglite-icu-full/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@electric-sql/pglite-icu-full",
+  "version": "0.0.1",
+  "description": "ICU support",
+  "author": "Electric DB Limited",
+  "homepage": "https://pglite.dev",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/electric-sql/pglite",
+    "directory": "packages/pglite-icu-full"
+  },
+  "keywords": [
+    "postgres",
+    "sql",
+    "database",
+    "wasm",
+    "pglite",
+    "initdb"
+  ],
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "./dist"
+  ],
+  "type": "module",
+  "types": "dist/index.d.ts",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    }
+  },
+  "scripts": {
+    "build": "tsup && cp static/* ./dist/",
+    "check:exports": "attw . --pack --profile node16",
+    "lint": "eslint ./tests --report-unused-disable-directives --max-warnings 0",
+    "format": "prettier --write ./tests",
+    "typecheck": "tsc",
+    "stylecheck": "pnpm lint && prettier --check ./tests",
+    "test": "vitest",
+    "prepublishOnly": "pnpm check:exports"
+  },
+  "devDependencies": {
+    "@arethetypeswrong/cli": "^0.18.1",
+    "@electric-sql/pglite": "workspace:*",
+    "@types/emscripten": "^1.41.1",
+    "@types/node": "^20.16.11",
+    "tsx": "^4.19.2",
+    "vitest": "^1.3.1",
+    "@electric-sql/pglite-utils": "workspace:*"
+  }
+}

packages/pglite-icu-full/src/index.ts

@@ -0,0 +1,13 @@
+import { pglUtils } from '@electric-sql/pglite-utils'
+
+export async function icuDataDir(): Promise<Blob> {
+  const moduleUrl = new URL('../dist/icu.76.tgz', import.meta.url)
+  if (pglUtils.IN_NODE) {
+    const fs = await import('fs/promises')
+    const buffer = await fs.readFile(moduleUrl)
+    return new Blob([new Uint8Array(buffer)])
+  } else {
+    const downloadPromise = await fetch(moduleUrl)
+    return downloadPromise.blob()
+  }
+}