Move build files to 'src', output sql files to '<db>/dist'

canadian_nf_mysql.sql → canadian_nf/dist/canadian_nf_mysql.sql

@@ -119,7 +119,7 @@ create table YLD_NM (
 );
 
 -- Load data into FOOD_NM
-load data local infile './canadian_nf/data/FOOD_NM.txt.trimmed'
+load data local infile '../data/FOOD_NM.txt.trimmed'
  into table FOOD_NM
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'
@@ -137,7 +137,7 @@ insert into tmp (select count(*) from tmp);
 drop table tmp;
 
 -- Load data into NT_AMT
-load data local infile './canadian_nf/data/NT_AMT.txt'
+load data local infile '../data/NT_AMT.txt'
  into table NT_AMT
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'
@@ -154,7 +154,7 @@ insert into tmp (select count(*) from tmp);
 drop table tmp;
 
 -- Load data into CONV_FAC
-load data local infile './canadian_nf/data/CONV_FAC.txt'
+load data local infile '../data/CONV_FAC.txt'
  into table CONV_FAC
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'
@@ -171,7 +171,7 @@ insert into tmp (select count(*) from tmp);
 drop table tmp;
 
 -- Load data into REFUSE
-load data local infile './canadian_nf/data/REFUSE.txt'
+load data local infile '../data/REFUSE.txt'
  into table REFUSE
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'
@@ -188,7 +188,7 @@ insert into tmp (select count(*) from tmp);
 drop table tmp;
 
 -- Load data into YIELD
-load data local infile './canadian_nf/data/YIELD.txt'
+load data local infile '../data/YIELD.txt'
  into table YIELD
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'
@@ -205,7 +205,7 @@ insert into tmp (select count(*) from tmp);
 drop table tmp;
 
 -- Load data into FOOD_GRP
-load data local infile './canadian_nf/data/FOOD_GRP.txt'
+load data local infile '../data/FOOD_GRP.txt'
  into table FOOD_GRP
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'
@@ -221,7 +221,7 @@ insert into tmp (select count(*) from tmp);
 drop table tmp;
 
 -- Load data into FOOD_SRC
-load data local infile './canadian_nf/data/FOOD_SRC.txt'
+load data local infile '../data/FOOD_SRC.txt'
  into table FOOD_SRC
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'
@@ -237,7 +237,7 @@ insert into tmp (select count(*) from tmp);
 drop table tmp;
 
 -- Load data into NT_NM
-load data local infile './canadian_nf/data/NT_NM.txt'
+load data local infile '../data/NT_NM.txt'
  into table NT_NM
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'
@@ -253,7 +253,7 @@ insert into tmp (select count(*) from tmp);
 drop table tmp;
 
 -- Load data into NT_SRC
-load data local infile './canadian_nf/data/NT_SRC.txt'
+load data local infile '../data/NT_SRC.txt'
  into table NT_SRC
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'
@@ -269,7 +269,7 @@ insert into tmp (select count(*) from tmp);
 drop table tmp;
 
 -- Load data into MEASURE
-load data local infile './canadian_nf/data/MEASURE.txt'
+load data local infile '../data/MEASURE.txt'
  into table MEASURE
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'
@@ -285,7 +285,7 @@ insert into tmp (select count(*) from tmp);
 drop table tmp;
 
 -- Load data into REFU_NM
-load data local infile './canadian_nf/data/REFU_NM.txt'
+load data local infile '../data/REFU_NM.txt'
  into table REFU_NM
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'
@@ -301,7 +301,7 @@ insert into tmp (select count(*) from tmp);
 drop table tmp;
 
 -- Load data into YLD_NM
-load data local infile './canadian_nf/data/YLD_NM.txt'
+load data local infile '../data/YLD_NM.txt'
  into table YLD_NM
  fields terminated by '$' optionally enclosed by '"'
  lines terminated by '\r\n'

canadian_nf_oracle.sql → canadian_nf/dist/canadian_nf_oracle.sql

@@ -144,7 +144,7 @@ CREATE TABLE YLD_NM (
 );
 
 -- Load data into FOOD_NM
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/FOOD_NM.ctl;
+HOST SQLLDR food/food control=../sqlldr/FOOD_NM.txt.trimmed;
 -- Assert all 5807 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);
@@ -154,7 +154,7 @@ INSERT INTO tmp (SELECT COUNT(*) FROM tmp);
 DROP TABLE tmp;
 
 -- Load data into NT_AMT
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/NT_AMT.ctl;
+HOST SQLLDR food/food control=../sqlldr/NT_AMT.ctl;
 -- Assert all 495878 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);
@@ -164,7 +164,7 @@ INSERT INTO tmp (SELECT COUNT(*) FROM tmp);
 DROP TABLE tmp;
 
 -- Load data into CONV_FAC
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/CONV_FAC.ctl;
+HOST SQLLDR food/food control=../sqlldr/CONV_FAC.ctl;
 -- Assert all 22260 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);
@@ -174,7 +174,7 @@ INSERT INTO tmp (SELECT COUNT(*) FROM tmp);
 DROP TABLE tmp;
 
 -- Load data into REFUSE
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/REFUSE.ctl;
+HOST SQLLDR food/food control=../sqlldr/REFUSE.ctl;
 -- Assert all 7138 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);
@@ -184,7 +184,7 @@ INSERT INTO tmp (SELECT COUNT(*) FROM tmp);
 DROP TABLE tmp;
 
 -- Load data into YIELD
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/YIELD.ctl;
+HOST SQLLDR food/food control=../sqlldr/YIELD.ctl;
 -- Assert all 1599 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);
@@ -194,7 +194,7 @@ INSERT INTO tmp (SELECT COUNT(*) FROM tmp);
 DROP TABLE tmp;
 
 -- Load data into FOOD_GRP
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/FOOD_GRP.ctl;
+HOST SQLLDR food/food control=../sqlldr/FOOD_GRP.ctl;
 -- Assert all 23 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);
@@ -204,7 +204,7 @@ INSERT INTO tmp (SELECT COUNT(*) FROM tmp);
 DROP TABLE tmp;
 
 -- Load data into FOOD_SRC
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/FOOD_SRC.ctl;
+HOST SQLLDR food/food control=../sqlldr/FOOD_SRC.ctl;
 -- Assert all 17 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);
@@ -214,7 +214,7 @@ INSERT INTO tmp (SELECT COUNT(*) FROM tmp);
 DROP TABLE tmp;
 
 -- Load data into NT_NM
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/NT_NM.ctl;
+HOST SQLLDR food/food control=../sqlldr/NT_NM.ctl;
 -- Assert all 150 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);
@@ -224,7 +224,7 @@ INSERT INTO tmp (SELECT COUNT(*) FROM tmp);
 DROP TABLE tmp;
 
 -- Load data into NT_SRC
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/NT_SRC.ctl;
+HOST SQLLDR food/food control=../sqlldr/NT_SRC.ctl;
 -- Assert all 20 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);
@@ -234,7 +234,7 @@ INSERT INTO tmp (SELECT COUNT(*) FROM tmp);
 DROP TABLE tmp;
 
 -- Load data into MEASURE
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/MEASURE.ctl;
+HOST SQLLDR food/food control=../sqlldr/MEASURE.ctl;
 -- Assert all 1174 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);
@@ -244,7 +244,7 @@ INSERT INTO tmp (SELECT COUNT(*) FROM tmp);
 DROP TABLE tmp;
 
 -- Load data into REFU_NM
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/REFU_NM.ctl;
+HOST SQLLDR food/food control=../sqlldr/REFU_NM.ctl;
 -- Assert all 151 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);
@@ -254,7 +254,7 @@ INSERT INTO tmp (SELECT COUNT(*) FROM tmp);
 DROP TABLE tmp;
 
 -- Load data into YLD_NM
-HOST SQLLDR food/food control=./canadian_nf/sqlldr/YLD_NM.ctl;
+HOST SQLLDR food/food control=../sqlldr/YLD_NM.ctl;
 -- Assert all 82 records were loaded
 CREATE TABLE tmp (c NUMBER PRIMARY KEY);
 INSERT INTO tmp (c) VALUES (2);

MySQL.pm → src/MySQL.pm

@@ -136,7 +136,8 @@ sub sql_load_file {
  my ($nutdbid, $user_name, $user_pwd, $file, $table_name, $field_separator, $text_separator, $line_separator, $ignore_header_lines, @fieldinfo) = @_;
 
  # TODO: how to make MySQL generate an error if varchar data truncation occurs?
- my $result = "load data local infile '$file'\n";
+ my $relative_file = join("", split /$nutdbid\//, $file);
+ my $result = "load data local infile '$relative_file'\n";
  $result .= " into table $table_name\n";
  $result .= " fields terminated by '$field_separator'";
  $result .= " optionally enclosed by '$text_separator'" if $text_separator;

Oracle.pm → src/Oracle.pm

@@ -154,16 +154,17 @@ sub sql_load_file {
  my ($nutdbid, $user_name, $user_pwd, $file, $table_name, $field_separator, $text_separator, $line_separator, $ignore_header_lines, @fieldinfo) = @_;
 
  # Keep things tidy and gather all control files into a subdir.
- `mkdir -p ./$nutdbid/sqlldr`;
+ `mkdir -p ../$nutdbid/sqlldr`;
 
  # Generate control file.
- $file = "./" . $nutdbid . "/sqlldr/" . $table_name . ".ctl";
+ $file =~ s|/data/|/sqlldr/|;
+ $file =~ s/\.txt$/\.ctl/;
  open FILE, ">$file" or die $!;
  print FILE "OPTIONS (DIRECT=TRUE, PARALLEL=TRUE"; # Load all or nothing.
  print FILE ", SKIP=$ignore_header_lines" if $ignore_header_lines;
  print FILE ")\n";
  print FILE "LOAD DATA\n";
- print FILE " INFILE './$nutdbid/data/$table_name.txt'\n"; # Using field separator so no need for streaming option to load multi-line values.
+ print FILE " INFILE '../data/$table_name.txt'\n"; # Using field separator so no need for streaming option to load multi-line values.
  print FILE " APPEND\n"; # Must be APPEND to use PARALLEL option.
  print FILE " INTO TABLE $table_name\n";
  print FILE " FIELDS TERMINATED BY '$field_separator'\n";
@@ -183,7 +184,8 @@ sub sql_load_file {
  close FILE;
 
  # Invoke sqlldr.
- return "HOST SQLLDR $user_name/$user_pwd control=$file;";
+ my $relative_file = join("", split /$nutdbid\//, $file);
+ return "HOST SQLLDR $user_name/$user_pwd control=$relative_file;";
 }
 
 sub sql_assert_record_count {

build.sh → src/build.sh

@@ -8,12 +8,12 @@ if [ "$PERL" = "" ]; then echo "Please install Perl" ; exit 1 ; fi
 ./clean.sh
 
 # Process all nutrient databases included.
-for NUTDBDIR in `find . -mindepth 1 -maxdepth 1 -type d`; do
+for NUTDBDIR in `find .. -mindepth 1 -maxdepth 1 -type d`; do
  # Extract nutrient dabatase identifier.
- NUTDBID=`expr "$NUTDBDIR" : "\./\(.*\)"`
+ NUTDBID=`expr "$NUTDBDIR" : "\.\./\(.*\)"`
 
- # Ignore .git dir.
- if [ "$NUTDBID" = ".git" ]; then continue; fi
+ # Ignore non-DB dirs.
+ if [ "$NUTDBID" = ".git" -o "$NUTDBID" = "src" ]; then continue; fi
 
  echo "================================== $NUTDBID =================================="
 
@@ -37,6 +37,6 @@ for NUTDBDIR in `find . -mindepth 1 -maxdepth 1 -type d`; do
  # Generate the SQL file for this database.
  # Make sure to add the current directory to the beginning of @INC
  # to avoid accidentally using official modules with the same name.
- $PERL -I . -M$RDBMSID ./generate_sql.pl $RDBMSID $NUTDBID $OUTFILE > $OUTFILE
+ $PERL -I . -M$RDBMSID ./generate_sql.pl $RDBMSID $NUTDBID $OUTFILE > ../$NUTDBID/dist/$OUTFILE
  done
 done

check_data_files.pl → src/check_data_files.pl

@@ -9,7 +9,7 @@
 my $prefix = $ARGV[1];
 die "Usage: $0 nutdbid [outputprefix]\n" if !$nutdbid;
 
-my @files = split /\n/, `file ./$nutdbid/data/*.txt`;
+my @files = split /\n/, `file ../$nutdbid/data/*.txt`;
 foreach (@files) {
  $_ =~ /^(.*):/;
  print $prefix . "WARNING: data file $1 contains non-ASCII characters\n" if $_ !~ /ASCII/;

clean.sh → src/clean.sh

@@ -1,8 +1,8 @@
 #!/bin/sh
 
 # Delete Oracle sqlldr control files.
-rm -rf */sqlldr
+rm -rf ../*/sqlldr
 
 # Delete trimmed files.
-find . -name \*.trimmed -exec rm {} \;
+find .. -name \*.trimmed -exec rm {} \;
 

generate_sql.pl → src/generate_sql.pl

@@ -23,7 +23,7 @@
 
 
 my $project_url = "http://github.com/m5n/nutriana";
-my $file = "./$nutdbid/schema.json";
+my $file = "../$nutdbid/schema.json";
 my $json = do { local $/ = undef; open my $fh, "<", $file or die "Could not open $file: $!"; <$fh>; };
 my $data = decode_json($json);
 my $header = $data->{"description"} . " (" . $data->{"url"} . ")";
@@ -80,7 +80,7 @@
  my %table = %{$_};
  my $table_name = substr($table{"file"}, 0, -length(".txt"));
  my @fieldinfo = ();
- my $datafile = "./$nutdbid/data/" . $table{"file"};
+ my $datafile = "../$nutdbid/data/" . $table{"file"};
  my $line_separator = "\\r\\n";
 
  # Convert files if needed.
@@ -94,7 +94,9 @@
  open OUTFILE, ">", $datafile or die "Could not open $datafile: $!";
  # TODO: doing this while loop ends up setting @{$data->{"tables"}}[0] to undefined, leading to trouble below. Why?
  while (<INFILE>) {
- $_ =~ s/\s$//g;
+print STDERR "BEFORE: $_ END\n" if /182.182/;
+ $_ =~ s/\s+$//g;
+print STDERR "AFTER: $_ END\n" if /182.182/;
  print OUTFILE $_ . "\r\n" if length($_) > 0; # Interpreted version of $line_separator. # TODO: find Perl function to convert "\\r\\n" -> "\r\n".
  }
  close INFILE;