Skip to content

chore(nodejs): regexp for table and column name validations #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 38 additions & 39 deletions src/sender.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,21 @@ const crypto = require('crypto');

const DEFAULT_BUFFER_SIZE = 8192;

const SPACE_REPLACE_REGEX = / /g;
const SPACE_REPLACE_VALUE = '\\ ';
const EQUALS_REPLACE_REGEX = /=/g;
const EQUALS_REPLACE_VALUE = '\\=';
const DOUBLE_QOUTE_REPLACE_REGEX = /"/g;
const DOUBLE_QOUTE_REPLACE_VALUE = '\\"';
const COMMA_REPLACE_REGEX = /,/g;
const COMMA_REPLACE_VALUE = '\\,';
const NEWLINE_REPLACE_REGEX = /\n/g;
const NEWLINE_REPLACE_VALUE = '\\\n';
const RETURN_REPLACE_REGEX = /\r/g;
const RETURN_REPLACE_VALUE = '\\\r';
const BACKSLASH_REPLACE_REGEX = /\\/g;
const BACKSLASH_REPLACE_VALUE = '\\\\';

/** @classdesc
* The QuestDB client's API provides methods to connect to the database, ingest data and close the connection.
* <p>
Expand Down Expand Up @@ -202,7 +217,7 @@ class Sender {
}
validateTableName(table);
checkCapacity(this, [table]);
writeEscaped(this, table);
writeName(this, table);
this.hasTable = true;
return this;
}
Expand All @@ -225,9 +240,9 @@ class Sender {
checkCapacity(this, [name, valueStr], 2 + name.length + valueStr.length);
write(this, ',');
validateColumnName(name);
writeEscaped(this, name);
writeName(this, name);
write(this, '=');
writeEscaped(this, valueStr);
writeNonQuotedValue(this, valueStr);
this.hasSymbols = true;
return this;
}
Expand All @@ -243,7 +258,7 @@ class Sender {
writeColumn(this, name, value, () => {
checkCapacity(this, [value], 2 + value.length);
write(this, '"');
writeEscaped(this, value, true);
writeQuotedValue(this, value);
write(this, '"');
}, "string");
return this;
Expand Down Expand Up @@ -417,7 +432,7 @@ function writeColumn(sender, name, value, writeValue, valueType) {
checkCapacity(sender, [name], 2 + name.length);
write(sender, sender.hasColumns ? ',' : ' ');
validateColumnName(name);
writeEscaped(sender, name);
writeName(sender, name);
write(sender, '=');
writeValue();
sender.hasColumns = true;
Expand All @@ -430,41 +445,25 @@ function write(sender, data) {
}
}

function writeEscaped(sender, data, quoted = false) {
for (const ch of data) {
if (ch > '\\') {
write(sender, ch);
continue;
}
function writeNonQuotedValue(sender, value) {
write(sender, value.replace(BACKSLASH_REPLACE_REGEX, BACKSLASH_REPLACE_VALUE)
.replace(SPACE_REPLACE_REGEX, SPACE_REPLACE_VALUE)
.replace(EQUALS_REPLACE_REGEX, EQUALS_REPLACE_VALUE)
.replace(COMMA_REPLACE_REGEX, COMMA_REPLACE_VALUE)
.replace(NEWLINE_REPLACE_REGEX, NEWLINE_REPLACE_VALUE)
.replace(RETURN_REPLACE_REGEX, RETURN_REPLACE_VALUE));
}

switch (ch) {
case ' ':
case ',':
case '=':
if (!quoted) {
write(sender, '\\');
}
write(sender, ch);
break;
case '\n':
case '\r':
write(sender, '\\');
write(sender, ch);
break;
case '"':
if (quoted) {
write(sender, '\\');
}
write(sender, ch);
break;
case '\\':
write(sender, '\\\\');
break;
default:
write(sender, ch);
break;
}
}
function writeQuotedValue(sender, value) {
write(sender, value.replace(BACKSLASH_REPLACE_REGEX, BACKSLASH_REPLACE_VALUE)
.replace(DOUBLE_QOUTE_REPLACE_REGEX, DOUBLE_QOUTE_REPLACE_VALUE)
.replace(NEWLINE_REPLACE_REGEX, NEWLINE_REPLACE_VALUE)
.replace(RETURN_REPLACE_REGEX, RETURN_REPLACE_VALUE));
}

function writeName(sender, name) {
write(sender, name.replace(SPACE_REPLACE_REGEX, SPACE_REPLACE_VALUE)
.replace(EQUALS_REPLACE_REGEX, EQUALS_REPLACE_VALUE));
}

exports.Sender = Sender;
112 changes: 24 additions & 88 deletions src/validation.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,15 @@

const QuestDBMaxFileNameLength = 127;

// eslint-disable-next-line no-control-regex
const INVALID_COLUMN_REGEX = /[?.,'"\\/:()+\-*%~\r\n\u{0000}\u{0001}\u{0002}\u{0003}\u{0004}\u{0005}\u{0006}\u{0007}\u{0008}\u{0009}\u{000B}\u{000C}\u{000E}\u{000F}\u{007F}\u{FEFF}]/u;
// eslint-disable-next-line no-control-regex
const INVALID_TABLE_REGEX = /[?,'"\\/:()+*%~\r\n\u{0000}\u{0001}\u{0002}\u{0003}\u{0004}\u{0005}\u{0006}\u{0007}\u{0008}\u{0009}\u{000B}\u{000C}\u{000E}\u{000F}\u{007F}\u{FEFF}]/u;
const INVALID_TABLE_START_DOT_REGEX = /^\./;
const INVALID_TABLE_END_DOT_REGEX = /\.$/;
const INVALID_TABLE_MORE_DOTS_REGEX = /\.\./;
const INVALID_DESIGNATED_REGEX = /\D/;

/**
* Validates a table name. <br>
* Throws an error if table name is invalid.
Expand All @@ -16,52 +25,17 @@ function validateTableName(name) {
if (len === 0) {
throw new Error("Empty string is not allowed as table name");
}
for (let i = 0; i < len; i++) {
let ch = name[i];
switch (ch) {
case '.':
if (i === 0 || i === len - 1 || name[i - 1] === '.')
// single dot is allowed in the middle only
// starting with a dot hides directory in Linux
// ending with a dot can be trimmed by some Windows versions / file systems
// double or triple dot looks suspicious
// single dot allowed as compatibility,
// when someone uploads 'file_name.csv' the file name used as the table name
throw new Error("Table name cannot start or end with a dot and only a single dot allowed");
break;
case '?':
case ',':
case '\'':
case '"':
case '\\':
case '/':
case ':':
case ')':
case '(':
case '+':
case '*':
case '%':
case '~':
case '\u0000':
case '\u0001':
case '\u0002':
case '\u0003':
case '\u0004':
case '\u0005':
case '\u0006':
case '\u0007':
case '\u0008':
case '\u0009': // control characters, except \n.
case '\u000B': // new line allowed for compatibility, there are tests to make sure it works
case '\u000c':
case '\r':
case '\n':
case '\u000e':
case '\u000f':
case '\u007f':
case '\ufeff': // UTF-8 BOM (Byte Order Mark) can appear at the beginning of a character stream
throw new Error(`Invalid character in table name: ${ch}`);
}
if (INVALID_TABLE_REGEX.test(name)) {
throw new Error(`Invalid character in table name: ${name}`);
}
if (INVALID_TABLE_START_DOT_REGEX.test(name)) {
throw new Error(`Table name cannot start with a dot: ${name}`);
}
if (INVALID_TABLE_END_DOT_REGEX.test(name)) {
throw new Error(`Table name cannot end with a dot: ${name}`);
}
if (INVALID_TABLE_MORE_DOTS_REGEX.test(name)) {
throw new Error(`Only single dots allowed in table name: ${name}`);
}
}

Expand All @@ -79,43 +53,8 @@ function validateColumnName(name) {
if (len === 0) {
throw new Error("Empty string is not allowed as column name");
}
for (const ch of name) {
switch (ch) {
case '?':
case '.':
case ',':
case '\'':
case '"':
case '\\':
case '/':
case ':':
case ')':
case '(':
case '+':
case '-':
case '*':
case '%':
case '~':
case '\u0000':
case '\u0001':
case '\u0002':
case '\u0003':
case '\u0004':
case '\u0005':
case '\u0006':
case '\u0007':
case '\u0008':
case '\u0009': // control characters, except \n
case '\u000B':
case '\u000c':
case '\r':
case '\n':
case '\u000e':
case '\u000f':
case '\u007f':
case '\ufeff': // UTF-8 BOM (Byte Order Mark) can appear at the beginning of a character stream
throw new Error(`Invalid character in column name: ${ch}`);
}
if (INVALID_COLUMN_REGEX.test(name)) {
throw new Error(`Invalid character in column name: ${name}`);
}
}

Expand All @@ -130,11 +69,8 @@ function validateDesignatedTimestamp(timestamp) {
if (len === 0) {
throw new Error("Empty string is not allowed as designated timestamp");
}
for (let i = 0; i < len; i++) {
let ch = timestamp[i];
if (ch < '0' || ch > '9') {
throw new Error(`Invalid character in designated timestamp: ${ch}`);
}
if (INVALID_DESIGNATED_REGEX.test(timestamp)) {
throw new Error(`Invalid character in designated timestamp: ${timestamp}`);
}
}

Expand Down
51 changes: 51 additions & 0 deletions test/flametest.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
'use strict';

const { Sender } = require("../index");
const { readFileSync } = require('fs');

const PORT = 9009;
const HOST = "127.0.0.1";

const PRIVATE_KEY = "9b9x5WhJywDEuo1KGQWSPNxtX-6X6R2BRCKhYMMY6n8";
const PUBLIC_KEY = {
x: "aultdA0PjhD_cWViqKKyL5chm6H1n-BiZBo_48T-uqc",
y: "__ptaol41JWSpTTL525yVEfzmY8A6Vi_QrW1FjKcHMg"
};
const JWK = {
...PUBLIC_KEY,
kid: "testapp",
kty: "EC",
d: PRIVATE_KEY,
crv: "P-256",
};

const senderTLS = {
host: HOST,
port: PORT,
ca: readFileSync('certs/ca/ca.crt') // necessary only if the server uses self-signed certificate
};

async function run() {
const tableName = "test";

const sender = new Sender({bufferSize: 131072, jwk: JWK});
await sender.connect(senderTLS);
const numOfRows = 300000;
for (let i = 0; i < numOfRows; i++) {
sender.table(tableName)
.symbol("location", `emea${i}`).symbol("city", `budapest${i}`)
.stringColumn("hoppa", `hello${i}`).stringColumn("hippi", `hel${i}`).stringColumn("hippo", `haho${i}`)
.floatColumn("temperature", 12.1).intColumn("intcol", i)
.atNow();
if (i % 1000 === 0) {
await sender.flush();
}
}
await sender.flush();
await sender.close();

return 0;
}

run().then(value => console.log(value))
.catch(err => console.log(err));
Loading