diff --git a/package.json b/package.json index fa6c436353..cf916508e6 100644 --- a/package.json +++ b/package.json @@ -90,4 +90,4 @@ "test" ] } -} \ No newline at end of file +} diff --git a/src/io/csv.js b/src/io/csv.js new file mode 100644 index 0000000000..4b1c2924fb --- /dev/null +++ b/src/io/csv.js @@ -0,0 +1,211 @@ +/* +The MIT License (MIT) + +Copyright (c) 2019 Evan Plaice + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ +export function parse (csv, options, reviver = v => v) { + const ctx = Object.create(null) + ctx.options = options || {} + ctx.reviver = reviver + ctx.value = '' + ctx.entry = [] + ctx.output = [] + ctx.col = 1 + ctx.row = 1 + + ctx.options.delimiter = ctx.options.delimiter === undefined ? '"' : options.delimiter; + if(ctx.options.delimiter.length > 1 || ctx.options.delimiter.length === 0) + throw Error(`CSVError: delimiter must be one character [${ctx.options.separator}]`) + + ctx.options.separator = ctx.options.separator === undefined ? ',' : options.separator; + if(ctx.options.separator.length > 1 || ctx.options.separator.length === 0) + throw Error(`CSVError: separator must be one character [${ctx.options.separator}]`) + + const lexer = new RegExp(`${escapeRegExp(ctx.options.delimiter)}|${escapeRegExp(ctx.options.separator)}|\r\n|\n|\r|[^${escapeRegExp(ctx.options.delimiter)}${escapeRegExp(ctx.options.separator)}\r\n]+`, 'y') + const isNewline = /^(\r\n|\n|\r)$/ + + let matches = [] + let match = '' + let state = 0 + + while ((matches = lexer.exec(csv)) !== null) { + match = matches[0] + + switch (state) { + case 0: // start of entry + switch (true) { + case match === ctx.options.delimiter: + state = 3 + break + case match === ctx.options.separator: + state = 0 + valueEnd(ctx) + break + case isNewline.test(match): + state = 0 + valueEnd(ctx) + entryEnd(ctx) + break + default: + ctx.value += match + state = 2 + break + } + break + case 2: // un-delimited input + switch (true) { + case match === ctx.options.separator: + state = 0 + valueEnd(ctx) + break + case isNewline.test(match): + state = 0 + valueEnd(ctx) + entryEnd(ctx) + break + default: + state = 4 + throw Error(`CSVError: Illegal state [row:${ctx.row}, col:${ctx.col}]`) + } + break + case 3: // delimited input + switch (true) { + case match === ctx.options.delimiter: + state = 4 + break + default: + state = 3 + ctx.value += match + break + } + break + case 4: // escaped or closing delimiter + switch (true) { + case match === ctx.options.delimiter: + state = 3 + ctx.value += match + break + case match === ctx.options.separator: + state = 0 + valueEnd(ctx) + break + case isNewline.test(match): + state = 0 + valueEnd(ctx) + entryEnd(ctx) + break + default: + throw Error(`CSVError: Illegal state [row:${ctx.row}, col:${ctx.col}]`) + } + break + } + } + + // flush the last value + if (ctx.entry.length !== 0) { + valueEnd(ctx) + entryEnd(ctx) + } + + return ctx.output +} + +export function stringify (array, options = {}, replacer = v => v) { + const ctx = Object.create(null) + ctx.options = options + ctx.options.eof = ctx.options.eof !== undefined ? ctx.options.eof : true + ctx.row = 1 + ctx.col = 1 + ctx.output = '' + + ctx.options.delimiter = ctx.options.delimiter === undefined ? '"' : options.delimiter; + if(ctx.options.delimiter.length > 1 || ctx.options.delimiter.length === 0) + throw Error(`CSVError: delimiter must be one character [${ctx.options.separator}]`) + + ctx.options.separator = ctx.options.separator === undefined ? ',' : options.separator; + if(ctx.options.separator.length > 1 || ctx.options.separator.length === 0) + throw Error(`CSVError: separator must be one character [${ctx.options.separator}]`) + + const needsDelimiters = new RegExp(`${escapeRegExp(ctx.options.delimiter)}|${escapeRegExp(ctx.options.separator)}|\r\n|\n|\r`) + + array.forEach((row, rIdx) => { + let entry = '' + ctx.col = 1 + row.forEach((col, cIdx) => { + if (typeof col === 'string') { + col = col.replace(new RegExp(ctx.options.delimiter, 'g'), `${ctx.options.delimiter}${ctx.options.delimiter}`) + col = needsDelimiters.test(col) ? `${ctx.options.delimiter}${col}${ctx.options.delimiter}` : col + } + entry += replacer(col, ctx.row, ctx.col) + if (cIdx !== row.length - 1) { + entry += ctx.options.separator + } + ctx.col++ + }) + switch (true) { + case ctx.options.eof: + case !ctx.options.eof && rIdx !== array.length - 1: + ctx.output += `${entry}\n` + break + default: + ctx.output += `${entry}` + break + } + ctx.row++ + }) + + return ctx.output +} + +function valueEnd (ctx) { + const value = ctx.options.typed ? inferType(ctx.value) : ctx.value + ctx.entry.push(ctx.reviver(value, ctx.row, ctx.col)) + ctx.value = '' + ctx.col++ +} + +function entryEnd (ctx) { + ctx.output.push(ctx.entry) + ctx.entry = [] + ctx.row++ + ctx.col = 1 +} + +function inferType (value) { + const isNumber = /.\./ + + switch (true) { + case value === 'true': + case value === 'false': + return value === 'true' + case isNumber.test(value): + return parseFloat(value) + case isFinite(value): + return parseInt(value) + default: + return value + } +} + +function escapeRegExp(str) { + return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&"); +} diff --git a/src/io/files.js b/src/io/files.js index bd588d4aa9..3834dbf6bc 100644 --- a/src/io/files.js +++ b/src/io/files.js @@ -8,6 +8,7 @@ import * as fileSaver from 'file-saver'; import { Renderer } from '../core/p5.Renderer'; import { Graphics } from '../core/p5.Graphics'; +import { parse } from './csv'; class HTTPError extends Error { status; @@ -530,18 +531,20 @@ function files(p5, fn){ try{ let { data } = await request(path, 'text'); - data = data.split(/\r?\n/); let ret = new p5.Table(); + data = parse(data, { + separator + }); if(header){ - ret.columns = data.shift().split(separator); + ret.columns = data.shift(); }else{ - ret.columns = data[0].split(separator).map(() => null); + ret.columns = Array(data[0].length).fill(null); } data.forEach((line) => { - const row = new p5.TableRow(line, separator); + const row = new p5.TableRow(line); ret.addRow(row); }); @@ -2032,40 +2035,8 @@ function files(p5, fn){ sep = '\t'; } if (ext !== 'html') { - // make header if it has values - if (header[0] !== '0') { - for (let h = 0; h < header.length; h++) { - if (h < header.length - 1) { - pWriter.write(header[h] + sep); - } else { - pWriter.write(header[h]); - } - } - pWriter.write('\n'); - } - - // make rows - for (let i = 0; i < table.rows.length; i++) { - let j; - for (j = 0; j < table.rows[i].arr.length; j++) { - if (j < table.rows[i].arr.length - 1) { - //double quotes should be inserted in csv only if contains comma separated single value - if (ext === 'csv' && String(table.rows[i].arr[j]).includes(',')) { - pWriter.write('"' + table.rows[i].arr[j] + '"' + sep); - } else { - pWriter.write(table.rows[i].arr[j] + sep); - } - } else { - //double quotes should be inserted in csv only if contains comma separated single value - if (ext === 'csv' && String(table.rows[i].arr[j]).includes(',')) { - pWriter.write('"' + table.rows[i].arr[j] + '"'); - } else { - pWriter.write(table.rows[i].arr[j]); - } - } - } - pWriter.write('\n'); - } + const output = table.toString(sep); + pWriter.write(output); } else { // otherwise, make HTML pWriter.print(''); diff --git a/src/io/p5.Table.js b/src/io/p5.Table.js index 7dca60a9bd..38c09df201 100644 --- a/src/io/p5.Table.js +++ b/src/io/p5.Table.js @@ -4,6 +4,8 @@ * @requires core */ +import { stringify } from './csv'; + function table(p5, fn){ /** * Table Options @@ -43,6 +45,18 @@ function table(p5, fn){ this.rows = []; } + toString(separator=',') { + let rows = this.rows.map((row) => row.arr); + + if(!this.columns.some((column) => column === null)){ + rows = [this.columns, ...rows,] + } + + return stringify(rows, { + separator + }); + } + /** * Use addRow() to add a new row of data to a p5.Table object. By default, * an empty row is created. Typically, you would store a reference to diff --git a/src/io/p5.TableRow.js b/src/io/p5.TableRow.js index 85bbf6cc61..f1590e0dcf 100644 --- a/src/io/p5.TableRow.js +++ b/src/io/p5.TableRow.js @@ -14,18 +14,12 @@ function tableRow(p5, fn){ * * @class p5.TableRow * @constructor - * @param {String} [str] optional: populate the row with a - * string of values, separated by the - * separator - * @param {String} [separator] comma separated values (csv) by default + * @param {any[]} row optional: populate the row with an + * array of values */ p5.TableRow = class { - constructor(str, separator){ - let arr = []; - if (str) { - separator = separator || ','; - arr = str.split(separator); - } + constructor(row=[]){ + let arr = row; this.arr = arr; this.obj = Object.fromEntries(arr.entries()); diff --git a/test/unit/io/loadTable.js b/test/unit/io/loadTable.js index de2b52803b..171d12f7c3 100644 --- a/test/unit/io/loadTable.js +++ b/test/unit/io/loadTable.js @@ -44,14 +44,14 @@ suite('loadTable', function() { test('returns an object with correct data', async () => { const table = await mockP5Prototype.loadTable(validFile); - assert.equal(table.getRowCount(), 5); + assert.equal(table.getRowCount(), 4); assert.strictEqual(table.getRow(1).getString(0), 'David'); assert.strictEqual(table.getRow(1).getNum(1), 31); }); test('passes an object with correct data to success callback', async () => { await mockP5Prototype.loadTable(validFile, (table) => { - assert.equal(table.getRowCount(), 5); + assert.equal(table.getRowCount(), 4); assert.strictEqual(table.getRow(1).getString(0), 'David'); assert.strictEqual(table.getRow(1).getNum(1), 31); }); @@ -59,32 +59,31 @@ suite('loadTable', function() { test('separator option returns the correct data', async () => { const table = await mockP5Prototype.loadTable(validFile, ','); - assert.equal(table.getRowCount(), 5); + assert.equal(table.getRowCount(), 4); assert.strictEqual(table.getRow(1).getString(0), 'David'); assert.strictEqual(table.getRow(1).getNum(1), 31); }); test('using the header option works', async () => { const table = await mockP5Prototype.loadTable(validFile, ',', true); - assert.equal(table.getRowCount(), 4); + assert.equal(table.getRowCount(), 3); assert.strictEqual(table.getRow(0).getString(0), 'David'); assert.strictEqual(table.getRow(0).getNum(1), 31); }); - test.todo('CSV files should handle commas within quoted fields', async () => { - // TODO: Current parsing does not handle quoted fields + test('CSV files should handle commas within quoted fields', async () => { const table = await mockP5Prototype.loadTable(validFile); - assert.equal(table.getRowCount(), 5); + assert.equal(table.getRowCount(), 4); assert.equal(table.getRow(2).get(0), 'David, Jr.'); assert.equal(table.getRow(2).getString(0), 'David, Jr.'); assert.equal(table.getRow(2).get(1), '11'); assert.equal(table.getRow(2).getString(1), 11); }); - test.todo('CSV files should handle escaped quotes and returns within quoted fields', async () => { + test('CSV files should handle escaped quotes and returns within quoted fields', async () => { // TODO: Current parsing does not handle quoted fields const table = await mockP5Prototype.loadTable(validFile); - assert.equal(table.getRowCount(), 5); + assert.equal(table.getRowCount(), 4); assert.equal(table.getRow(3).get(0), 'David,\nSr. "the boss"'); }); }); diff --git a/test/unit/io/saveTable.js b/test/unit/io/saveTable.js index fb41726dab..d6edf15fe8 100644 --- a/test/unit/io/saveTable.js +++ b/test/unit/io/saveTable.js @@ -15,7 +15,7 @@ suite('saveTable', function() { files(mockP5, mockP5Prototype); table(mockP5, mockP5Prototype); tableRow(mockP5, mockP5Prototype); - myTable = await mockP5Prototype.loadTable(validFile, 'csv', 'header'); + myTable = await mockP5Prototype.loadTable(validFile, ',', 'header'); }); afterEach(() => {