Skip to content

Feature: Custom SQL support #569

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions demos/example-vite-custom-sql/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# example-vite-custom-sql

## 0.0.1
9 changes: 9 additions & 0 deletions demos/example-vite-custom-sql/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# PowerSync Vite custom SQL demo

This is a minimal example demonstrating how to use custom SQL with PowerSync.

To see it in action:

1. Make sure to run `pnpm install` and `pnpm build:packages` in the root directory of this repo.
2. `cd` into this directory, and run `pnpm start`.
3. Open the localhost URL displayed in the terminal output in your browser.
22 changes: 22 additions & 0 deletions demos/example-vite-custom-sql/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{
"name": "example-vite-custom-sql",
"private": true,
"version": "0.0.1",
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build",
"preview": "vite preview",
"start": "pnpm build && pnpm preview",
"test:build": "pnpm build"
},
"dependencies": {
"@powersync/web": "workspace:*"
},
"devDependencies": {
"@swc/core": "~1.6.0",
"vite": "^5.0.12",
"vite-plugin-top-level-await": "^1.4.1",
"vite-plugin-wasm": "^3.3.0"
}
}
9 changes: 9 additions & 0 deletions demos/example-vite-custom-sql/src/index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<!doctype html>
<html>
<head>
<script type="module" src="./index.js"></script>
</head>
<body>
Custom SQL demo: Check the console to see it in action!
</body>
</html>
88 changes: 88 additions & 0 deletions demos/example-vite-custom-sql/src/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import { column, Schema, Table, PowerSyncDatabase } from '@powersync/web';
import Logger from 'js-logger';

Logger.useDefaults();

/**
* A placeholder connector which doesn't do anything.
* This is just used to verify that the sync workers can be loaded
* when connecting.
*/
class DummyConnector {
async fetchCredentials() {
return {
endpoint: '',
token: ''
};
}

async uploadData(database) {}
}

const customers = new Table({ first_name: column.text, last_name: column.text, full_name: column.text });

export const AppSchema = new Schema({ customers }, ({ getInternalName }) => [
`DROP TRIGGER IF EXISTS compute_full_name`,
`DROP TRIGGER IF EXISTS update_full_name`,

`
CREATE TRIGGER compute_full_name
AFTER INSERT ON ${getInternalName('customers')}
BEGIN
UPDATE customers
SET full_name = first_name || ' ' || last_name
WHERE id = NEW.id;
END;
`,
`
CREATE TRIGGER update_full_name
AFTER UPDATE OF data ON ${getInternalName('customers')}
BEGIN
UPDATE customers
SET full_name = first_name || ' ' || last_name
WHERE id = NEW.id AND full_name != (first_name || ' ' || last_name);
END;
`
]);

let PowerSync;

const openDatabase = async () => {
PowerSync = new PowerSyncDatabase({
schema: AppSchema,
database: { dbFilename: 'test.sqlite' }
});

await PowerSync.init();

await PowerSync.execute('DELETE FROM customers');

await PowerSync.execute('INSERT INTO customers(id, first_name, last_name) VALUES(uuid(), ?, ?)', ['John', 'Doe']);

const result = await PowerSync.getAll('SELECT * FROM customers');

console.log('Contents of customers after insert: ', result);

await PowerSync.execute('UPDATE customers SET first_name = ?', ['Jane']);

const result2 = await PowerSync.getAll('SELECT * FROM customers');

console.log('Contents of customers after update: ', result2);

console.log(
`Attempting to connect in order to verify web workers are correctly loaded.
This doesn't use any actual network credentials.
Network errors will be shown: these can be ignored.`
);

/**
* Try and connect, this will setup shared sync workers
* This will fail due to not having a valid endpoint,
* but it will try - which is all that matters.
*/
await PowerSync.connect(new DummyConnector());
};

document.addEventListener('DOMContentLoaded', (event) => {
openDatabase();
});
27 changes: 27 additions & 0 deletions demos/example-vite-custom-sql/vite.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import wasm from 'vite-plugin-wasm';
import topLevelAwait from 'vite-plugin-top-level-await';
import { defineConfig } from 'vite';

// https://vitejs.dev/config/
export default defineConfig({
root: 'src',
build: {
outDir: '../dist',
rollupOptions: {
input: 'src/index.html'
},
emptyOutDir: true
},
envDir: '..', // Use this dir for env vars, not 'src'.
optimizeDeps: {
// Don't optimize these packages as they contain web workers and WASM files.
// https://github.com/vitejs/vite/issues/11672#issuecomment-1415820673
exclude: ['@journeyapps/wa-sqlite', '@powersync/web'],
include: ['@powersync/web > js-logger']
},
plugins: [wasm(), topLevelAwait()],
worker: {
format: 'es',
plugins: () => [wasm(), topLevelAwait()]
}
});
39 changes: 24 additions & 15 deletions packages/common/src/client/AbstractPowerSyncDatabase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,15 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
this._schema = schema;

await this.database.execute('SELECT powersync_replace_schema(?)', [JSON.stringify(this.schema.toJSON())]);
try {
for await (const sql of this.schema.getCustomSQL?.({
getInternalName: (tableName) => this.schema.getTableInternalName(tableName) ?? tableName
}) ?? []) {
await this.database.execute(sql);
}
} catch (ex) {
this.options.logger?.error('Error executing custom SQL', ex);
}
await this.database.refreshSchema();
this.iterateListeners(async (cb) => cb.schemaChanged?.(schema));
}
Expand Down Expand Up @@ -555,7 +564,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
* This method does include transaction ids in the result, but does not group
* data by transaction. One batch may contain data from multiple transactions,
* and a single transaction may be split over multiple batches.
*
*
* @param limit Maximum number of CRUD entries to include in the batch
* @returns A batch of CRUD operations to upload, or null if there are none
*/
Expand Down Expand Up @@ -594,7 +603,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
*
* Unlike {@link getCrudBatch}, this only returns data from a single transaction at a time.
* All data for the transaction is loaded into memory.
*
*
* @returns A transaction of CRUD operations to upload, or null if there are none
*/
async getNextCrudTransaction(): Promise<CrudTransaction | null> {
Expand Down Expand Up @@ -633,7 +642,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
* Get an unique client id for this database.
*
* The id is not reset when the database is cleared, only when the database is deleted.
*
*
* @returns A unique identifier for the database instance
*/
async getClientId(): Promise<string> {
Expand Down Expand Up @@ -661,7 +670,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
/**
* Execute a SQL write (INSERT/UPDATE/DELETE) query
* and optionally return results.
*
*
* @param sql The SQL query to execute
* @param parameters Optional array of parameters to bind to the query
* @returns The query result as an object with structured key-value pairs
Expand All @@ -674,7 +683,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
/**
* Execute a SQL write (INSERT/UPDATE/DELETE) query directly on the database without any PowerSync processing.
* This bypasses certain PowerSync abstractions and is useful for accessing the raw database results.
*
*
* @param sql The SQL query to execute
* @param parameters Optional array of parameters to bind to the query
* @returns The raw query result from the underlying database as a nested array of raw values, where each row is
Expand All @@ -689,7 +698,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
* Execute a write query (INSERT/UPDATE/DELETE) multiple times with each parameter set
* and optionally return results.
* This is faster than executing separately with each parameter set.
*
*
* @param sql The SQL query to execute
* @param parameters Optional 2D array of parameter sets, where each inner array is a set of parameters for one execution
* @returns The query result
Expand All @@ -701,7 +710,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB

/**
* Execute a read-only query and return results.
*
*
* @param sql The SQL query to execute
* @param parameters Optional array of parameters to bind to the query
* @returns An array of results
Expand All @@ -713,7 +722,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB

/**
* Execute a read-only query and return the first result, or null if the ResultSet is empty.
*
*
* @param sql The SQL query to execute
* @param parameters Optional array of parameters to bind to the query
* @returns The first result if found, or null if no results are returned
Expand All @@ -725,7 +734,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB

/**
* Execute a read-only query and return the first result, error if the ResultSet is empty.
*
*
* @param sql The SQL query to execute
* @param parameters Optional array of parameters to bind to the query
* @returns The first result matching the query
Expand Down Expand Up @@ -761,7 +770,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
* Open a read-only transaction.
* Read transactions can run concurrently to a write transaction.
* Changes from any write transaction are not visible to read transactions started before it.
*
*
* @param callback Function to execute within the transaction
* @param lockTimeout Time in milliseconds to wait for a lock before throwing an error
* @returns The result of the callback
Expand All @@ -786,7 +795,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
* Open a read-write transaction.
* This takes a global lock - only one write transaction can execute against the database at a time.
* Statements within the transaction must be done on the provided {@link Transaction} interface.
*
*
* @param callback Function to execute within the transaction
* @param lockTimeout Time in milliseconds to wait for a lock before throwing an error
* @returns The result of the callback
Expand Down Expand Up @@ -865,7 +874,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
* Source tables are automatically detected using `EXPLAIN QUERY PLAN`.
*
* Note that the `onChange` callback member of the handler is required.
*
*
* @param sql The SQL query to execute
* @param parameters Optional array of parameters to bind to the query
* @param handler Callbacks for handling results and errors
Expand Down Expand Up @@ -915,7 +924,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
* Execute a read query every time the source tables are modified.
* Use {@link SQLWatchOptions.throttleMs} to specify the minimum interval between queries.
* Source tables are automatically detected using `EXPLAIN QUERY PLAN`.
*
*
* @param sql The SQL query to execute
* @param parameters Optional array of parameters to bind to the query
* @param options Options for configuring watch behavior
Expand Down Expand Up @@ -944,7 +953,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
* Resolves the list of tables that are used in a SQL query.
* If tables are specified in the options, those are used directly.
* Otherwise, analyzes the query using EXPLAIN to determine which tables are accessed.
*
*
* @param sql The SQL query to analyze
* @param parameters Optional parameters for the SQL query
* @param options Optional watch options that may contain explicit table list
Expand Down Expand Up @@ -1077,7 +1086,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
*
* This is preferred over {@link watchWithAsyncGenerator} when multiple queries need to be performed
* together when data is changed.
*
*
* Note: do not declare this as `async *onChange` as it will not work in React Native.
*
* @param options Options for configuring watch behavior
Expand Down
9 changes: 8 additions & 1 deletion packages/common/src/db/schema/Schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ export class Schema<S extends SchemaType = SchemaType> {
readonly props: S;
readonly tables: Table[];

constructor(tables: Table[] | S) {
constructor(
tables: Table[] | S,
readonly getCustomSQL?: ({ getInternalName }: { getInternalName: (tableName: string) => string }) => string[]
) {
if (Array.isArray(tables)) {
/*
We need to validate that the tables have a name here because a user could pass in an array
Expand Down Expand Up @@ -64,4 +67,8 @@ export class Schema<S extends SchemaType = SchemaType> {
return convertedTable;
});
}

getTableInternalName(tableName: string) {
return this.tables.find((t) => t.name === tableName)?.internalName;
}
}
10 changes: 10 additions & 0 deletions packages/common/tests/db/schema/Schema.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,4 +110,14 @@ describe('Schema', () => {
]
});
});

it('should get the internal name of a table', () => {
const schema = new Schema({
users: new Table({ name: column.text }),
posts: new Table({ name: column.text })
});

expect(schema.getTableInternalName('users')).toBe('ps_data__users');
expect(schema.getTableInternalName('posts')).toBe('ps_data__posts');
});
});