Implemented the loading of the influenza_aa.dat file. Tested by

comparing an export of the data from the HDF5 file and the original
file.

4 files changed, 145 insertions, 41 deletions

diff --git a/README b/README
index 9caedb8..197d289 100644
--- a/README
+++ b/README
@@ -32,4 +32,32 @@ The "doc/Data Deployments.dia" diagram shows the source systems that
 expose the various records as well as the transform routines that are
 used for aggregation of the data on the local system.
 
- LocalWords:  NCBI parallelization HDF SQL Pellegrino phylogenetic DBMS dia
+BUILDING
+
+An autogen.sh script is provided to initialize the project directory
+with the necessary GNU Autotools configuration.
+
+When building on a Debian system the mpi.h file is in a subdirectory
+of /usr/include and therefore not found within the default include
+path.  To account for this run the following before running
+./configure.
+
+  $ export CPPFLAGS=-I/usr/include/mpi
+
+TEST CASES
+
+The "load_influenza_aa_dat" function loads a single tab delimited text
+file into a table structure in the HDF5 file.  The HDFView GUI can be
+used to open the loaded table and then export it back out as a text
+file.  The text file can then be compared with the original input to
+verify that the load was completed without error.
+
+  $ diff --report-identical-files \
+    /home/don/exp004/genomes/INFLUENZA/influenza_aa.dat \
+    Protein\ Sequences.txt 
+
+  Files /home/don/exp004/genomes/INFLUENZA/influenza_aa.dat and
+  Protein Sequences.txt are identical
+
+ LocalWords:  NCBI parallelization HDF SQL Pellegrino phylogenetic DBMS dia mpi
+ LocalWords:  autogen Autotools CPPFLAGS aa dat HDFView GUI diff txt
diff --git a/doc/Data Deployments.dia b/doc/Data Deployments.dia
index b8ad4af..277d53a 100644
--- a/doc/Data Deployments.dia
+++ b/doc/Data Deployments.dia
diff --git a/src/aggregator.c b/src/aggregator.c
index ae5aa60..da6db08 100644
--- a/src/aggregator.c
+++ b/src/aggregator.c
@@ -24,7 +24,7 @@ main()
   /*
    * Close the HD5 file.
    */
-  herr_t status = H5Fclose (file_id);
+  H5Fclose (file_id);
 
   return 0;
 }
diff --git a/src/load_influenza_aa_dat.c b/src/load_influenza_aa_dat.c
index 72aacb5..5af8a72 100644
--- a/src/load_influenza_aa_dat.c
+++ b/src/load_influenza_aa_dat.c
@@ -1,10 +1,17 @@
+/*
+ * Load the influnza_aa.dat tab delimited text file into an HDF5
+ * binary table.
+ *
+ * todo: Handle NULL values occuring in numeric fields.
+ */
+
 #include "load_influenza_aa_dat.h"
 #include "hdf5_hl.h"
+#include <string.h>
+#include <stdlib.h>
 
 #define NFIELDS (hsize_t) 11
-//#define NRECORDS (hsize_t) 138052
-#define NRECORDS (hsize_t) 1
-#define TABLE_NAME "influenza_aa.dat"
+#define TABLE_NAME "Protein Sequences"
 
 void
 load_influenza_aa_dat (hid_t file_id)
@@ -12,14 +19,14 @@ load_influenza_aa_dat (hid_t file_id)
   /*
    * Model the data using native types.
    */
-  typedef struct supplementary_data
+  typedef struct
   {
     char genbank_accession_number[9];
     char host[15];
     int genome_segment_number;
     char subtype[7];
     char country[25];
-    int year;
+    char year[8];
     int sequence_length;
     char virus_name[196];
     char age[17];
@@ -49,11 +56,7 @@ load_influenza_aa_dat (hid_t file_id)
 				 HOFFSET ( supplementary_data, gender ),
 				 HOFFSET ( supplementary_data, full_length_indicator )};
 
-  /*
-
-    Only needed for reading?
-
-  supplementary_data dst_buf[NRECORDS];
+  supplementary_data dst_buf[1];
 
   size_t dst_sizes[NFIELDS] = { sizeof ( dst_buf[0].genbank_accession_number ),
 				sizeof ( dst_buf[0].host ),
@@ -66,29 +69,9 @@ load_influenza_aa_dat (hid_t file_id)
 				sizeof ( dst_buf[0].age ),
 				sizeof ( dst_buf[0].gender ),
 				sizeof ( dst_buf[0].full_length_indicator)};
-  */
 
   /*
-   * "Define field information."
-   */
-  const char *field_names[NFIELDS] =
-    { "GenBank accession number",
-      "Host",
-      "Genome segment number",
-      "Subtype",
-      "Country",
-      "Year",
-      "Sequence length",
-      "Virus name",
-      "Age",
-      "Gender",
-      "Full-length Indicator" };
-  hsize_t chunk_size = 10;
-  int *fill_data = NULL;
-  int compress = 0;
-  
-  /*
-   * "Initialize field type."
+   * Map the native types to HDF5 types for each field.
    */
   hid_t field_type[NFIELDS];
 
@@ -110,7 +93,9 @@ load_influenza_aa_dat (hid_t file_id)
   H5Tset_size (country_type, 25 );
   field_type[4] = country_type;
 
-  field_type[5] = H5T_NATIVE_INT; 
+  hid_t year_type = H5Tcopy ( H5T_C_S1 );
+  H5Tset_size (year_type, 8);
+  field_type[5] = year_type;
 
   field_type[6] = H5T_NATIVE_INT;
 
@@ -130,19 +115,110 @@ load_influenza_aa_dat (hid_t file_id)
   H5Tset_size (full_length_indicator_type, 4);
   field_type[10] = full_length_indicator_type;
 
-  supplementary_data p_data[NRECORDS] = {
-    {"BAC53999", "Human", 7, "", "Zambia", 1999, 109, "Influenza B virus (B/Lusaka/270/99)",
-     "", "", "yes"}
-  };
+  /*
+   * Labels used for the fields in the table.
+   */
+  const char *field_names[NFIELDS] =
+    { "GenBank accession number",
+      "Host",
+      "Genome segment number",
+      "Subtype",
+      "Country",
+      "Year",
+      "Sequence length",
+      "Virus name",
+      "Age",
+      "Gender",
+      "Full-length Indicator" };
+
+  /*
+   * Table storage options.
+   */
+  hsize_t chunk_size = 10;
+  int *fill_data = NULL;
+  int compress = 0;
+
+  /*
+   * Insert the records.
+   */
+  supplementary_data p_data;
+  FILE* dat = fopen ("/home/don/exp004/genomes/INFLUENZA/influenza_aa.dat", "r");
+  char *line = NULL;
+  size_t len = 0;
+  int current_line = 0;
+
+  while (getline (&line, &len, dat) != -1) {
+
+    current_line++;
+    char *running = strdup (line);
+    char *token;
+    
+    /*
+     * Parse the line, handling the case of empty fields represented
+     * by sequential delimiters.
+     */
+    strncpy(p_data.genbank_accession_number, strsep (&running, "\t"),
+	    sizeof(p_data.genbank_accession_number));
+    
+    strncpy(p_data.host, strsep (&running, "\t"),
+	    sizeof(p_data.host));
+    
+    token = strsep (&running, "\t");
+    if (strcmp (token, "\0") == 0)
+      p_data.genome_segment_number = 0;
+    else
+      p_data.genome_segment_number = atoi(token);
+    
+    strncpy(p_data.subtype, strsep (&running, "\t"),
+	    sizeof(p_data.subtype));
+    
+    strncpy(p_data.country, strsep (&running, "\t"),
+	    sizeof(p_data.country));
+    
+    strncpy (p_data.year, strsep (&running, "\t"),
+	     sizeof(p_data.year));
+
+    token = strsep (&running, "\t");
+    if (strcmp (token, "\0") == 0)
+      p_data.sequence_length = 0;
+    else
+      p_data.sequence_length = atoi(token);
+    
+    strncpy(p_data.virus_name, strsep (&running, "\t"),
+	    sizeof(p_data.virus_name));
+    
+    strncpy(p_data.age, strsep (&running, "\t"),
+	    sizeof(p_data.age));
+    
+    strncpy(p_data.gender, strsep (&running, "\t"),
+	    sizeof(p_data.gender));
+    
+    strncpy(p_data.full_length_indicator, strsep (&running, "\t"),
+	    sizeof(p_data.full_length_indicator));
+
+    if (current_line == 1)     
+      H5TBmake_table ("Protein Sequences", file_id, TABLE_NAME,NFIELDS,1,
+		      dst_size,field_names, dst_offset, field_type,
+		      chunk_size, fill_data, compress, &p_data);
+    else     
+      H5TBappend_records (file_id, TABLE_NAME, 1, dst_size, dst_offset,
+			 dst_sizes, &p_data);
+
+    if (running)
+      free (running);
+   
+  }
+  
+  if (line)
+    free (line);
 
-  herr_t status = H5TBmake_table ("influenza_aa.dat", file_id, TABLE_NAME,NFIELDS,NRECORDS,
-				  dst_size,field_names, dst_offset, field_type,
-				  chunk_size, fill_data, compress, p_data);
+  fclose (dat);
 
   H5Tclose (genbank_accession_number_type);
   H5Tclose (host_type);
   H5Tclose (subtype_type);
   H5Tclose (country_type);
+  H5Tclose (year_type);
   H5Tclose (virus_name_type);
   H5Tclose (age_type);
   H5Tclose (gender_type);