more validation work + more perl wrapper work

Author: bruceravel

c/README.md

@@ -49,6 +49,66 @@ contents of the XDI file along with a few particularly important items
 
 ## API
 
+The API is intended to separate, as much as possible, the chores of
+parsing the file and validating its content.
+
+The main point of entry is the function `XDI_readfile` which creates
+an `XDIFile` struct, then opens and parses a file for XDI content.
+This function will signal an error and issue a useful error message in
+the even that some content of the file precludes its being understood
+as XDI data.  These situations include such things as
+
+ * Missing XDI versioning information in the first line of the file
+ * Non-numeric content in the data table
+ * Variable number of columns in the data table
+ * Headers that cannot be interpreted as XDI metadata
+
+Certain conditions that are non-conforming with the XDI specification
+but which do not preclude partial interpretation of the XDI file
+result in warning messages at this time.
+
+Validating the content of the XDI metadata is treated as a separate
+step.  There are three main functions for this purpose:
+
+* `XDI_required_metadata`: This function tests that the pieces of
+  metadata which are **required** by the specification are present in
+  the file and intepretable.  These three metadata items are
+  `Mono.d_spacing`, `Element.symbol`, and `Element.edge`.  If not
+  present, this function returns a non-zero error code and issues an
+  explanatory error message.
+
+* `XDI_recommended_metadata`: This function tests that the pieces of
+  metadata which are **recommended** by the specification are present
+  in the file and intepretable.  The absence of these items does not
+  make the file non-compliant with the XDI specification, but their
+  absence impacts the utility and quality of the data.  If not
+  present, this function returns a non-zero warning code and issues an
+  explanatory error message.
+
+* `XDI_validate_item`: This tests a specific metadata item to see if
+  its value is interpretable according to its description in the XDI
+  dictionary.  Not all items defined in the dictionary have validation
+  tests.  If the value does not conform to its dictionary definition a
+  non-zero warning code is returned and an explanatory error message
+  is issued.  The purpose of this validation tool is to aid in
+  automated quality assurance.  For example, data by a user for
+  consideration in a curated standards database might require that all
+  automated validation tests pass before the data are passed along to
+  a human for further consideration.
+
+All error messages are returned in English as the content of the
+`error_message` attribute of the `XDIFile` struct.  The
+`error_message` attribute always contains a description of the error
+condition of the most recently performed action.  The relation between
+the returned error/warning codes and teh error messages are tabulated
+below.
+
+The value of separating most validation chores from the parsing of the
+file is that it allows an application to choose to accept structurally
+compliant data then to perform the level of validation appropriate to
+the task at hand.
+
+
 ### Read an XDI file
 
 Read an XDI file, store it's content in an XDIFile struct, and return
@@ -59,7 +119,7 @@ an integer return code.
 	int ret;
 
 	xdifile = malloc(sizeof(XDIFile));
-	ret = XDI_readfile("mydata.xdi", xdifile);
+	ret = XDI_readfile(xdifile, "mydata.xdi");
 ```
 
 ### Interpret the XDI_readfile error code
@@ -163,6 +223,20 @@ table.  The return value is -1 if the array cannot be retrieved.
 
 The array names are held in the `Column.N` metadata fields.
 
+### Extract indexed arrays from the data table
+
+```C
+	double *enarray, *muarray;
+	enarray = (double *)calloc(xdifile->npts, sizeof(double));
+	muarray = (double *)calloc(xdifile->npts, sizeof(double));
+    ret = XDI_get_array_index(xdifile, 1, enarray);
+    ret = XDI_get_array_index(xdifile, 2, muarray);
+```
+
+The return value is 0 if an array with that index is in the data
+table.  That is, the index argument must be smaller than the `narrays`
+attribute. The return value is -1 if the array cannot be retrieved.
+
 ### Destroy and deallocate the XDIFile struct
 
 To deallocate the memory from the XDIFile struct, do this:

c/xdifile.c

@@ -570,16 +570,22 @@ XDI_readfile(char *filename, XDIFile *xdifile) {
 /* ============================================================================ */
 /* array management section                                                     */
 
+#define ENOUGH ((8 * sizeof(int) - 1) / 3 + 2)
+
 _EXPORT(int)
 XDI_get_array_index(XDIFile *xdifile, long n, double *out) {
   /* get array by index (starting at 0) from an XDIFile structure */
   long j;
+  char str[ENOUGH];
   if (n < xdifile->narrays) {
     for (j = 0; j < xdifile->npts; j++) {
       out[j] = xdifile->array[n][j];
     }
     return 0;
   }
+  strcpy(xdifile->error_message, "no array of index ");
+  sprintf(str, "%ld", n);
+  strcat(xdifile->error_message, str);
   return -1;
 }
 
@@ -592,6 +598,8 @@ XDI_get_array_name(XDIFile *xdifile, char *name, double *out) {
       return XDI_get_array_index(xdifile, i, out);
     }
   }
+  strcpy(xdifile->error_message, "no array of name ");
+  strcat(xdifile->error_message, name);
   return -1;
 }
 
@@ -780,6 +788,9 @@ int XDI_validate_mono(XDIFile *xdifile, char *name, char *value) {
   return err;
 }
 
+/* all tests in the Sample familty should return WRN_BAD_SAMPLE if the */
+/* value does not match the definition.  error_message should explain  */
+/* the problem in a way that is appropruiate to the metadata item */
 int XDI_validate_sample(XDIFile *xdifile, char *name, char *value) {
   int err;
   int regex_status;

c/xdifile.h

@@ -155,5 +155,4 @@ static char *ValidElems[] =
 #define ERR_NONNUMERIC      -32	/* used */
 #define ERR_MEMERROR        -64	/* NOT used */
 
-
 /* _EXPORT(char*) XDI_errorstring(int errcode); */

perl/lib/Xray/XDI.pm

@@ -25,10 +25,7 @@ has 'xdifile' => (
 		  builder   => '_build_object',
 		 );
 # no need to fiddle with inline_constructor here
-has 'ok'             => (is => 'rw', isa => 'Bool',     traits => [qw(NoClone)], default => 0);
-has 'warning'        => (is => 'rw', isa => 'Bool',     traits => [qw(NoClone)], default => 0);
 has 'errorcode'      => (is => 'rw', isa => 'Int',      traits => [qw(NoClone)], default => 0);
-has 'error'          => (is => 'rw', isa => 'Str',      traits => [qw(NoClone)], default => q{});
 has 'errormessage'   => (is => 'rw', isa => 'Str',      traits => [qw(NoClone)], default => q{});
 
 has 'filename'       => (is => 'rw', isa => 'Str',      traits => [qw(NoClone)], default => q{});
@@ -83,30 +80,24 @@ sub DEMOLISH {
 sub _build_object {
   my ($self) = @_;
   $self->errormessage(q{});
-  $self->ok(1);
-  $self->warning(0);
   if (not $self->file) {
     $self->errorcode(1);
     $self->errormessage('No file specified');
-    $self->ok(0);
     return undef;
   };
   if (not -e $self->file) {
     $self->errorcode(1);
     $self->errormessage('The file '.$self->file.' does not exist');
-    $self->ok(0);
     return undef;
   };
   if (not -r $self->file) {
     $self->errorcode(1);
     $self->errormessage('The file '.$self->file.' cannot be read');
-    $self->ok(0);
     return undef;
   };
   if (-d $self->file) {
     $self->errorcode(1);
     $self->errormessage($self->file.' is a folder (i.e. not an XDI file)');
-    $self->ok(0);
     return undef;
   };
   my $errcode = 0;
@@ -116,34 +107,8 @@ sub _build_object {
 
   return $obj if ($errcode < 0);
 
-  #print $self->file, $/;
-  #$self->trace;
-  #print '>>>>>', $errcode, $/;
-
-  ##### see xdifile.h for error codes
-  ##### see xdifile.c (line 23 and following) for error messages
-  # if ($errcode < 0) {
-  #   my @errors = ();
-  #   foreach my $i (0 .. 10) {
-  #     push @errors, $obj->_errorstring(-1*2**$i) if (abs($errcode) & 2**$i);
-  #   };
-  #   $self->error(join(", ", @errors));
-  #   $self->ok(0);
-  #   return $obj;
-  # };
-  # if ($errcode > 0) {
-  #   my @errors = ();
-  #   foreach my $i (0 .. 4) {
-  #     push @errors, $obj->_errorstring(2**$i) if ($errcode & 2**$i);
-  #   };
-  #   $self->error(join(", ", @errors));
-  #   $self->ok(1);
-  #   $self->warning(1);
-  # };
-
   if (not defined $obj->_filename) {
-    $self->error('unknown problem reading '.$self->file.' as an XDI file');
-    $self->ok(0);
+    $self->errormessage('unknown problem reading '.$self->file.' as an XDI file');
     return $obj;
   };
 
@@ -356,10 +321,10 @@ Import an XDI file:
 
   use Xray::XDI;
   my $xdi = Xray::XDI->new(file=>'data.dat');
-  if ($xdi->ok) {
+  if ($xdi->errorcode == 0) {
     # do stuff
   } else {
-    print "Uh oh! ", $xdi->error, $/;
+    print "Uh oh! ", $xdi->errormessage, $/;
   };
 
 Export an XDI file:
@@ -376,16 +341,17 @@ The fully resolved path to the XDI file.  Setting this triggers the
 importing of the file and the setting of all other attributes from the
 contents of the file.
 
-=item C<ok>
+=item C<errormessage>
 
-This is true when C<file> is properly imported.  When false, the
-problem will be recorded in the C<error> attribute.
+When an XDI file is imported properly, this is an empty string.  When
+import or validation runs into a problem, the explanation will be
+stroed here as a string.
 
-=item C<error>
+=item C<errorcode>
 
-When an XDI file is imported properly, this is an empty string.  When
-import runs into a problem, the explanation will be stroed here as a
-string.
+The numeric code returned by C<libxdifile> when a problem is
+encountered.  The C<errorcode> always corresponds to the
+C<errormessage>.
 
 =item C<xdifile>
 
@@ -411,6 +377,10 @@ The element of the absorber.
 
 The absorption edge at which the data were measured.
 
+=item C<dspacing>
+
+The d-spacing (or line spacing) of the mono used in the measurement.
+
 =item C<comments>
 
 The user supplied comments with all white space and carriage returns
@@ -467,6 +437,31 @@ Alternately,
   ## later....
   $xdi -> file('/path/to/file');
 
+=item C<required>
+
+Test the XDIFile object for whether it contains the metadata items
+B<required> by the XDI spec, C<Mono.d_spacing>, C<Element.symbol>, and
+C<Element.edge>.
+
+  my $code = $xdi->required;
+  print $xdi->error_message, $/ if ($code);
+
+=item C<recommended>
+
+Test the XDIFile object for whether it contains the metadata items
+B<recommended> by the XDI spec.
+
+  my $code = $xdi->recommended;
+  print $xdi->error_message, $/ if ($code);
+
+=item C<validate>
+
+Validate a sepcific metadata item against its definition in the XDI
+dictionary.
+
+  my $code = $xdi->validate($family, $name, $value);
+  print $xdi->error_message, $/ if ($code);
+
 =item C<labels>
 
 Return a list of column labels in the order of appearance in the XDI
@@ -567,17 +562,6 @@ in brackets.
 
 =back
 
-=head1 VALIDATION
-
-If the sole intent is to validate an XDI file, i.e. to determine
-whether or not it conforms to the specification, it should be adequate
-to do something like the following:
-
-  $xdi = Xray::XDI->new('somefile.dat');
-  $is_valid = $xdi->ok;
-  $problem = $xdi->error if not $is_valid;
-  undef $xdi;
-  print "okee dokee!\n" if $is_valid;
 
 =head1 DIAGNOSTICS
 
@@ -608,6 +592,11 @@ L<Moose>, L<MooseX::NonMoose>
 
 =over 4
 
+=iten *
+
+need a validate method that reads, runs required and recommend, and
+validates all metadata, returning true if no problems found.
+
 =item *
 
 need an add data column method

perl/t/01_moose_nonmoose.t

@@ -17,8 +17,8 @@ my $here = dirname($0);
 my $file = File::Spec->catfile($here, '..', '..', 'data', 'co_metal_rt.xdi');
 my $xdi  = Xray::XDI->new(file=>$file);
 ok($xdi =~ m{Xray::XDI},                           'created Xray::XDI object');
-ok($xdi->ok,                                       'ok flag is true');
-ok($xdi->error eq '',                              'error text is empty');
+ok(($xdi->errorcode == 0),                         'errorcode 0');
+ok(($xdi->errormessage =~ m{\A\s*\z}),             'error text is empty');
 
 ok( $xdi->token('comment') eq '#',                 'token: comment');
 ok( $xdi->token('delimiter') eq ':',               'token: delimiter');

perl/t/02_moosish.t

@@ -17,7 +17,7 @@ my $here = dirname($0);
 my $file = File::Spec->catfile($here, '..', '..', 'data', 'co_metal_rt.xdi');
 my $xdi  = Xray::XDI->new(file=>$file);
 
-ok($xdi->ok,                                                      'file imported properly');
+ok($xdi->errorcode == 0,                                           'file imported properly');
 
 ##### test things that return arrays of strings #################
 

perl/t/gooddata/01_cu_metal_10k.t

@@ -17,7 +17,7 @@ my $here = dirname($0);
 my $file = File::Spec->catfile($here, '..', '..', '..', 'data', 'cu_metal_10K.xdi');
 my $xdi  = Xray::XDI->new(file=>$file);
 
-ok($xdi->ok,                                                      'file imported properly');
+ok($xdi->errorcode == 0,                                          'file imported properly');
 
 ##### test things that return arrays of strings #################
 

perl/t/gooddata/02_cu_metal_rt.t

@@ -17,7 +17,7 @@ my $here = dirname($0);
 my $file = File::Spec->catfile($here, '..', '..', '..', 'data', 'cu_metal_rt.xdi');
 my $xdi  = Xray::XDI->new(file=>$file);
 
-ok($xdi->ok,                                                      'file imported properly');
+ok($xdi->errorcode == 0,                                          'file imported properly');
 
 ##### test things that return arrays of strings #################
 

perl/t/gooddata/03_fe2o3.t

@@ -17,7 +17,7 @@ my $here = dirname($0);
 my $file = File::Spec->catfile($here, '..', '..', '..', 'data', 'fe2o3_rt.xdi');
 my $xdi  = Xray::XDI->new(file=>$file);
 
-ok($xdi->ok,                                                      'file imported properly');
+ok($xdi->errorcode == 0,                                          'file imported properly');
 
 ##### test things that return arrays of strings #################
 

perl/t/gooddata/04_fe3c_rt.t

@@ -17,7 +17,7 @@ my $here = dirname($0);
 my $file = File::Spec->catfile($here, '..', '..', '..', 'data', 'fe3c_rt.xdi');
 my $xdi  = Xray::XDI->new(file=>$file);
 
-ok($xdi->ok,                                                      'file imported properly');
+ok($xdi->errorcode == 0,                                          'file imported properly');
 
 ##### test things that return arrays of strings #################