Verify that CSV input is encoded as UTF-8

han-tun · Dec 6, 2017 · afb5cec · afb5cec
1 parent f3e6afa
commit afb5cec
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 1 deletion.
diff --git a/Makefile b/Makefile
@@ -59,7 +59,7 @@ tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/json
 tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
-tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o
+tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 unit: unit.o text.o

diff --git a/csv.cpp b/csv.cpp
@@ -1,4 +1,5 @@
 #include "csv.hpp"
+#include "text.hpp"
 
 std::vector<std::string> csv_split(const char *s) {
 	std::vector<std::string> ret;
@@ -67,13 +68,25 @@ void readcsv(const char *fn, std::vector<std::string> &header, std::map<std::str
 
 	std::string s;
 	if ((s = csv_getline(f)).size() > 0) {
+		std::string err = check_utf8(s);
+		if (err != "") {
+			fprintf(stderr, "%s: %s\n", fn, err.c_str());
+			exit(EXIT_FAILURE);
+		}
+
 		header = csv_split(s.c_str());
 
 		for (size_t i = 0; i < header.size(); i++) {
 			header[i] = csv_dequote(header[i]);
 		}
 	}
 	while ((s = csv_getline(f)).size() > 0) {
+		std::string err = check_utf8(s);
+		if (err != "") {
+			fprintf(stderr, "%s: %s\n", fn, err.c_str());
+			exit(EXIT_FAILURE);
+		}
+
 		std::vector<std::string> line = csv_split(s.c_str());
 		if (line.size() > 0) {
 			line[0] = csv_dequote(line[0]);

diff --git a/geocsv.cpp b/geocsv.cpp
@@ -20,6 +20,12 @@ void parse_geocsv(std::vector<struct serialization_state> &sst, std::string fnam
 	ssize_t latcol = -1, loncol = -1;
 
 	if ((s = csv_getline(f)).size() > 0) {
+		std::string err = check_utf8(s);
+		if (err != "") {
+			fprintf(stderr, "%s: %s\n", fname.c_str(), err.c_str());
+			exit(EXIT_FAILURE);
+		}
+
 		header = csv_split(s.c_str());
 
 		for (size_t i = 0; i < header.size(); i++) {
@@ -41,6 +47,12 @@ void parse_geocsv(std::vector<struct serialization_state> &sst, std::string fnam
 
 	size_t seq = 0;
 	while ((s = csv_getline(f)).size() > 0) {
+		std::string err = check_utf8(s);
+		if (err != "") {
+			fprintf(stderr, "%s: %s\n", fname.c_str(), err.c_str());
+			exit(EXIT_FAILURE);
+		}
+
 		seq++;
 		std::vector<std::string> line = csv_split(s.c_str());
 

diff --git a/jsontool.cpp b/jsontool.cpp
@@ -9,6 +9,7 @@
 #include <vector>
 #include "jsonpull/jsonpull.h"
 #include "csv.hpp"
+#include "text.hpp"
 
 int fail = EXIT_SUCCESS;
 bool wrap = false;
@@ -208,6 +209,12 @@ void join_csv(json_object *j) {
 			exit(EXIT_FAILURE);
 		}
 
+		std::string err = check_utf8(s);
+		if (err != "") {
+			fprintf(stderr, "%s\n", err.c_str());
+			exit(EXIT_FAILURE);
+		}
+
 		header = csv_split(s.c_str());
 
 		for (size_t i = 0; i < header.size(); i++) {
@@ -264,6 +271,12 @@ void join_csv(json_object *j) {
 				break;
 			}
 
+			std::string err = check_utf8(s);
+			if (err != "") {
+				fprintf(stderr, "%s\n", err.c_str());
+				exit(EXIT_FAILURE);
+			}
+
 			fields = csv_split(s.c_str());
 
 			for (size_t i = 0; i < fields.size(); i++) {