fix(tables): persist coerced values in upsert match + bulk update, normalize Date to ISO

Author: TheodoreSpeaks

apps/sim/lib/table/__tests__/service-filter-threading.test.ts

@@ -35,6 +35,7 @@ vi.mock('@/lib/table/validation', () => ({
   validateRowSize: vi.fn(() => ({ valid: true, errors: [] })),
   validateRowAgainstSchema: vi.fn(() => ({ valid: true, errors: [] })),
   coerceRowToSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  coerceRowValues: vi.fn(),
   validateTableName: vi.fn(() => ({ valid: true, errors: [] })),
   validateTableSchema: vi.fn(() => ({ valid: true, errors: [] })),
   getUniqueColumns: vi.fn(() => []),

apps/sim/lib/table/__tests__/update-row.test.ts

@@ -21,6 +21,7 @@ vi.mock('@/lib/table/validation', () => ({
   validateRowSize: vi.fn(() => ({ valid: true, errors: [] })),
   validateRowAgainstSchema: vi.fn(() => ({ valid: true, errors: [] })),
   coerceRowToSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  coerceRowValues: vi.fn(),
   validateTableName: vi.fn(() => ({ valid: true, errors: [] })),
   validateTableSchema: vi.fn(() => ({ valid: true, errors: [] })),
   getUniqueColumns: vi.fn(() => []),

apps/sim/lib/table/__tests__/validation.test.ts

@@ -6,6 +6,7 @@ import { TABLE_LIMITS } from '../constants'
 import {
   type ColumnDefinition,
   coerceRowToSchema,
+  coerceRowValues,
   getUniqueColumns,
   type TableSchema,
   validateColumnDefinition,
@@ -334,6 +335,14 @@ describe('Validation', () => {
       expect(data.created).toBe(new Date(epoch).toISOString())
     })
 
+    it('coerces a Date instance to an ISO date string', () => {
+      const date = new Date('2024-01-15T00:00:00Z')
+      const data = { name: 'Acme', founded: 2000, created: date }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(true)
+      expect(data.created).toBe(date.toISOString())
+    })
+
     it('leaves already-correct values untouched and passes through json', () => {
       const data = { name: 'Acme', founded: 2000, metadata: { k: 'v' } }
       const result = coerceRowToSchema(data, schema)
@@ -349,6 +358,34 @@ describe('Validation', () => {
     })
   })
 
+  describe('coerceRowValues', () => {
+    const schema: TableSchema = {
+      columns: [
+        { name: 'name', type: 'string', required: true },
+        { name: 'founded', type: 'number', required: true },
+        { name: 'age', type: 'number' },
+      ],
+    }
+
+    it('coerces a partial patch in place without flagging absent required fields', () => {
+      const patch = { age: '42' }
+      coerceRowValues(patch, schema)
+      expect(patch.age).toBe(42)
+    })
+
+    it('nulls an un-coercible optional value in a patch', () => {
+      const patch: { age: unknown } = { age: 'nope' }
+      coerceRowValues(patch as never, schema)
+      expect(patch.age).toBeNull()
+    })
+
+    it('leaves an un-coercible required value in place for downstream validation', () => {
+      const patch: { founded: unknown } = { founded: 'nope' }
+      coerceRowValues(patch as never, schema)
+      expect(patch.founded).toBe('nope')
+    })
+  })
+
   describe('getUniqueColumns', () => {
     it('should return only columns with unique=true', () => {
       const schema: TableSchema = {

apps/sim/lib/table/service.ts

@@ -62,6 +62,7 @@ import {
   checkBatchUniqueConstraintsDb,
   checkUniqueConstraintsDb,
   coerceRowToSchema,
+  coerceRowValues,
   getUniqueColumns,
   validateRowSize,
   validateTableName,
@@ -1331,11 +1332,6 @@ export async function upsertRow(
     )
   }
 
-  const targetValue = data.data[targetColumnName]
-  if (targetValue === undefined || targetValue === null) {
-    throw new Error(`Upsert requires a value for the conflict target column "${targetColumnName}"`)
-  }
-
   // Validate row data
   const sizeValidation = validateRowSize(data.data)
   if (!sizeValidation.valid) {
@@ -1347,6 +1343,13 @@ export async function upsertRow(
     throw new Error(`Schema validation failed: ${schemaValidation.errors.join(', ')}`)
   }
 
+  // Read the conflict-target value *after* coercion so `matchFilter` branches on
+  // the persisted type (e.g. a coerced `"123"` → `123` matches existing rows).
+  const targetValue = data.data[targetColumnName]
+  if (targetValue === undefined || targetValue === null) {
+    throw new Error(`Upsert requires a value for the conflict target column "${targetColumnName}"`)
+  }
+
   // `data->` and `data->>` accept the JSON key as a parameterized text value;
   // no need for `sql.raw` interpolation.
   const matchFilter =
@@ -2167,6 +2170,12 @@ export async function updateRowsByFilter(
     return { affectedCount: 0, affectedRowIds: [] }
   }
 
+  // Coerce the patch itself in place — the write below persists `data.data`
+  // (as `patchJson`), so coercing only the per-row merged copies would be
+  // discarded. The merged validation in the loop still enforces required
+  // fields against the full row.
+  coerceRowValues(data.data, table.schema)
+
   for (const row of matchingRows) {
     const existingData = row.data as RowData
     const mergedData = { ...existingData, ...data.data }

apps/sim/lib/table/validation.ts

@@ -290,7 +290,7 @@ function coerceValueToColumnType(
       }
       return { ok: false }
     case 'date':
-      if (value instanceof Date) return { ok: true, value }
+      if (value instanceof Date) return { ok: true, value: value.toISOString() }
       if (typeof value === 'string' && !Number.isNaN(Date.parse(value))) return { ok: true, value }
       if (typeof value === 'number' && Number.isFinite(value)) {
         return { ok: true, value: new Date(value).toISOString() }
@@ -302,18 +302,16 @@ function coerceValueToColumnType(
 }
 
 /**
- * Coerces each value in `data` toward its column's declared type **in place**,
- * then validates the result. Values that already match are untouched;
- * unambiguous conversions (e.g. `"1999"` → `1999`) are applied; values that
- * cannot be coerced are set to `null` when the column is optional, or left in
- * place to fail validation when the column is required.
+ * Coerces each present value in `data` toward its column's declared type **in
+ * place**. Values that already match are untouched; unambiguous conversions
+ * (e.g. `"1999"` → `1999`) are applied; values that cannot be coerced are set to
+ * `null` when the column is optional, or left in place when required (so a
+ * subsequent {@link validateRowAgainstSchema} reports them).
  *
- * This is the write-path entry point — callers that persist rows use it instead
- * of {@link validateRowAgainstSchema} so a single off-type field (a tool
- * returning `"unknown"` for a numeric column, say) nulls that one cell rather
- * than failing the entire row write.
+ * Operates per-present-column, so it is safe on a partial patch (columns absent
+ * from `data` are skipped — it never invents a missing-required-field error).
  */
-export function coerceRowToSchema(data: RowData, schema: TableSchema): ValidationResult {
+export function coerceRowValues(data: RowData, schema: TableSchema): void {
   for (const column of schema.columns) {
     const value = data[column.name]
     if (value === null || value === undefined) continue
@@ -325,7 +323,21 @@ export function coerceRowToSchema(data: RowData, schema: TableSchema): Validatio
       data[column.name] = null
     }
   }
+}
 
+/**
+ * Coerces a full row toward its schema **in place** (see {@link coerceRowValues})
+ * then validates the result.
+ *
+ * This is the write-path entry point — callers that persist a complete row use
+ * it instead of {@link validateRowAgainstSchema} so a single off-type field (a
+ * tool returning `"unknown"` for a numeric column, say) nulls that one cell
+ * rather than failing the entire row write. Callers persisting only a partial
+ * patch should use {@link coerceRowValues} on the patch and validate the merged
+ * row separately.
+ */
+export function coerceRowToSchema(data: RowData, schema: TableSchema): ValidationResult {
+  coerceRowValues(data, schema)
   return validateRowAgainstSchema(data, schema)
 }