Merge pull request #17 from asdf-format/issue-15

Implement lazy YAML parsing

Author: embray

Makefile.am

@@ -1,9 +1,9 @@
-AM_CFLAGS = # CFLAGS applicable to all executables (products)
+AM_CFLAGS = -fvisibility=hidden
 AM_CPPFLAGS = -I$(top_srcdir)/src # so that tests also find header files
 ACLOCAL_AMFLAGS = -Im4
 
-src_files = src/block.c src/event.c src/info.c src/parse.c src/yaml.c
-src_headers = src/block.h src/event.h src/info.h src/parse.h src/yaml.h
+src_files = src/block.c src/event.c src/info.c src/parse.c src/parse_util.c src/yaml.c
+src_headers = src/block.h src/event.h src/info.h src/parse.h src/parse_util.h src/yaml.h
 
 noinst_HEADERS = $(src_headers)
 
@@ -37,8 +37,7 @@ tidy:
 
 CODE_COVERAGE_IGNORE_PATTERN = \
     $(abs_top_srcdir)/tests/* \
-    $(abs_top_srcdir)/tests/munit/* \
-    /usr/include/x86_64-linux-gnu/bits/*
+    $(abs_top_srcdir)/tests/munit/*
 
 # For rules generated by AX_AM_MACROS_STATIC; e.g. code coverage rules
 include $(top_srcdir)/aminclude_static.am

src/event.c

@@ -23,6 +23,22 @@ asdf_event_type_t asdf_event_type(asdf_event_t *event) {
 }
 
 
+const char *asdf_event_comment(const asdf_event_t *event) {
+    if (!(event && event->type == ASDF_COMMENT_EVENT))
+        return NULL;
+
+    return event->payload.comment;
+}
+
+
+const asdf_tree_info_t *asdf_event_tree_info(const asdf_event_t *event) {
+    if (!(event && (event->type == ASDF_TREE_START_EVENT || event->type == ASDF_TREE_END_EVENT)))
+        return NULL;
+
+    return event->payload.tree;
+}
+
+
 const asdf_block_info_t *asdf_event_block_info(const asdf_event_t *event) {
     if (!(event && event->type == ASDF_BLOCK_EVENT))
         return NULL;
@@ -67,6 +83,11 @@ void asdf_event_print(const asdf_event_t *event, FILE *file, bool verbose) {
         fprintf(file, "  Standard Version: %s\n", event->payload.version->version);
         break;
 
+
+    case ASDF_COMMENT_EVENT:
+        fprintf(file, "  Comment: %s\n", event->payload.comment);
+        break;
+
     case ASDF_YAML_EVENT: {
         fprintf(file, "  Type: %s\n", asdf_yaml_event_type_text(event));
 
@@ -85,14 +106,29 @@ void asdf_event_print(const asdf_event_t *event, FILE *file, bool verbose) {
         break;
     }
 
+    case ASDF_TREE_START_EVENT:
+        fprintf(file, "  Tree start position: %zu (0x%zx)\n", event->payload.tree->start,
+                event->payload.tree->start);
+        break;
+
+    case ASDF_TREE_END_EVENT:
+        fprintf(file, "  Tree end position: %zu (0x%zx)\n", event->payload.tree->end,
+                event->payload.tree->end);
+        break;
+
     case ASDF_BLOCK_EVENT: {
         const asdf_block_info_t *block = event->payload.block;
         const asdf_block_header_t header = block->header;
-        fprintf(file, "  Header position: %" PRId64 "\n", (int64_t)block->header_pos);
-        fprintf(file, "  Data position: %" PRId64 "\n", (int64_t)block->data_pos);
-        fprintf(file, "  Allocated size: %" PRIu64 "\n", header.allocated_size);
-        fprintf(file, "  Used size: %" PRIu64 "\n", header.used_size);
-        fprintf(file, "  Data size: %" PRIu64 "\n", header.data_size);
+        fprintf(file, "  Header position: %" PRId64 " (0x%" PRIx64 ")\n",
+                (int64_t)block->header_pos, (int64_t)block->header_pos);
+        fprintf(file, "  Data position: %" PRId64 " (0x%" PRIx64 ")\n",
+                (int64_t)block->data_pos, (int64_t)block->data_pos);
+        fprintf(file, "  Allocated size: %" PRIu64 " (0x%" PRIx64 ")\n",
+                header.allocated_size, header.allocated_size);
+        fprintf(file, "  Used size: %" PRIu64 " (0x%" PRIx64 ")\n", header.used_size,
+                header.used_size);
+        fprintf(file, "  Data size: %" PRIu64 " (0x%" PRIx64 ")\n", header.data_size,
+                header.data_size);
 
         if (header.compression[0] != '\0')
             fprintf(file, "  Compression: %.4s\n", header.compression);
@@ -114,6 +150,13 @@ void asdf_event_print(const asdf_event_t *event, FILE *file, bool verbose) {
 void asdf_event_destroy(asdf_parser_t *parser, asdf_event_t *event) {
     assert(event);
     switch (event->type) {
+    case ASDF_TREE_START_EVENT:
+    case ASDF_TREE_END_EVENT:
+        if (event->payload.tree)
+            free(event->payload.tree->buf);
+
+        free(event->payload.tree);
+        break;
     case ASDF_YAML_EVENT:
         fy_parser_event_free(parser->yaml_parser, event->payload.yaml);
         break;
@@ -127,6 +170,9 @@ void asdf_event_destroy(asdf_parser_t *parser, asdf_event_t *event) {
     case ASDF_BLOCK_EVENT:
         free(event->payload.block);
         break;
+    case ASDF_COMMENT_EVENT:
+        free(event->payload.comment);
+        break;
     default:
         break;
     }

src/event.h

@@ -9,17 +9,19 @@
 
 #include "block.h"
 #include "parse.h"
+#include "util.h"
 #include "yaml.h"
 
 
 #define ASDF_EVENT_TYPES(X) \
     X(ASDF_NONE_EVENT) \
     X(ASDF_BEGIN_EVENT) \
-    X(ASDF_COMMENT_EVENT) \
     X(ASDF_ASDF_VERSION_EVENT) \
     X(ASDF_STANDARD_VERSION_EVENT) \
-    X(ASDF_YAML_VERSION_EVENT) \
+    X(ASDF_COMMENT_EVENT) \
+    X(ASDF_TREE_START_EVENT) \
     X(ASDF_YAML_EVENT) \
+    X(ASDF_TREE_END_EVENT) \
     X(ASDF_BLOCK_EVENT) \
     X(ASDF_PADDING_EVENT) \
     X(ASDF_BLOCK_INDEX_EVENT) \
@@ -47,20 +49,36 @@ typedef struct {
 } asdf_version_t;
 
 
+typedef struct {
+    size_t start;
+    size_t end;
+    char *buf;
+} asdf_tree_info_t;
+
+
 typedef struct asdf_event {
     asdf_event_type_t type;
     union {
-        asdf_block_info_t *block;
-        asdf_yaml_event_t *yaml;
+        // Only if ASDF_*_VERSION_EVENT
         asdf_version_t *version;
+        // Only if ASDF_COMMENT_EVENT
+        char *comment;
+        // Only if ASDF_TREE_*_EVENT
+        asdf_tree_info_t *tree;
+        // Only if ASDF_YAML_EVENT
+        asdf_yaml_event_t *yaml;
+        // Only if ASDF_BLOCK_EVENT
+        asdf_block_info_t *block;
     } payload;
 } asdf_event_t;
 
 
 /* Public API functions */
-asdf_event_type_t asdf_event_type(asdf_event_t *event);
-const asdf_block_info_t *asdf_event_block_info(const asdf_event_t *event);
-int asdf_event_iterate(asdf_parser_t *parser, asdf_event_t *event);
-const char *asdf_event_type_name(asdf_event_type_t event_type);
-void asdf_event_print(const asdf_event_t *event, FILE *file, bool verbose);
-void asdf_event_destroy(asdf_parser_t *parser, asdf_event_t *event);
+ASDF_EXPORT asdf_event_type_t asdf_event_type(asdf_event_t *event);
+ASDF_EXPORT const char *asdf_event_comment(const asdf_event_t *event);
+ASDF_EXPORT const asdf_tree_info_t *asdf_event_tree_info(const asdf_event_t *event);
+ASDF_EXPORT const asdf_block_info_t *asdf_event_block_info(const asdf_event_t *event);
+ASDF_EXPORT int asdf_event_iterate(asdf_parser_t *parser, asdf_event_t *event);
+ASDF_EXPORT const char *asdf_event_type_name(asdf_event_type_t event_type);
+ASDF_EXPORT void asdf_event_print(const asdf_event_t *event, FILE *file, bool verbose);
+ASDF_EXPORT void asdf_event_destroy(asdf_parser_t *parser, asdf_event_t *event);

src/info.c

@@ -459,7 +459,12 @@ int asdf_info(FILE *in_file, FILE *out_file, const asdf_info_cfg_t *cfg) {
     if (!cfg)
         cfg = &ASDF_INFO_DEFAULT_CFG;
 
-    if (asdf_parser_init(&parser) != 0)
+    // Current implementation needs YAML events unless no-tree
+    asdf_parser_cfg_t parser_cfg = {
+        .flags = cfg->print_tree ? ASDF_PARSER_OPT_EMIT_YAML_EVENTS : 0
+    };
+
+    if (asdf_parser_init(&parser, &parser_cfg) != 0)
         return 1;
 
     if (asdf_parser_set_input_file(&parser, in_file, cfg->filename) != 0) {

src/info.h

@@ -6,6 +6,8 @@
 
 #include <stdio.h>
 
+#include "util.h"
+
 
 /**
  * Optional configuration options for the `asdf_info` function
@@ -17,4 +19,4 @@ typedef struct {
 } asdf_info_cfg_t;
 
 
-int asdf_info(FILE *in_file, FILE *out_file, const asdf_info_cfg_t *cfg);
+ASDF_EXPORT int asdf_info(FILE *in_file, FILE *out_file, const asdf_info_cfg_t *cfg);

src/main.c

@@ -151,13 +151,18 @@ static char events_doc[] = "Print event stream from ASDF parser (for debugging)"
 static char events_args_doc[] = "FILENAME";
 
 
+#define EVENTS_OPT_NO_YAML_KEY 0x100
+
+
 static struct argp_option events_options[] = {
-    {"verbose", 'v', 0, 0, "Show extra information about each event", 0}, {0}};
+    {"verbose", 'v', 0, 0, "Show extra information about each event", 0},
+    {"no-yaml", EVENTS_OPT_NO_YAML_KEY, 0, 0, "Do not produce YAML stream events", 0}, {0}};
 
 
 struct events_args {
     const char *filename;
     bool verbose;
+    bool no_yaml;
 };
 
 
@@ -166,6 +171,9 @@ static error_t parse_events_opt(int key, char *arg, struct argp_state *state) {
     struct events_args *args = state->input;
 
     switch (key) {
+    case EVENTS_OPT_NO_YAML_KEY:
+        args->no_yaml = true;
+        break;
     case 'v':
         args->verbose = true;
         break;
@@ -190,7 +198,7 @@ static error_t parse_events_opt(int key, char *arg, struct argp_state *state) {
 static struct argp events_argp = {events_options, parse_events_opt, events_args_doc, events_doc};
 
 
-int events_main(const char *filename, bool verbose) {
+int events_main(const char *filename, bool verbose, bool no_yaml) {
     FILE *file = fopen(filename, "r");
 
     if (!file) {
@@ -199,8 +207,11 @@ int events_main(const char *filename, bool verbose) {
     }
 
     asdf_parser_t parser;
+    // Current implementation always outputs YAML events, so this is needed; later update
+    // to allow an option to skip YAML events (useful for testing)
+    asdf_parser_cfg_t parser_cfg = {.flags = no_yaml ? 0 : ASDF_PARSER_OPT_EMIT_YAML_EVENTS};
 
-    if (asdf_parser_init(&parser)) {
+    if (asdf_parser_init(&parser, &parser_cfg)) {
         fprintf(stderr, "error: %s\n", asdf_parser_get_error(&parser));
         fclose(file);
         return EXIT_FAILURE;
@@ -253,7 +264,7 @@ int main(int argc, char *argv[]) {
         argp_parse(
             &events_argp, global_args.subcmd_argc, global_args.subcmd_argv, 0, NULL, &events_args);
 
-        return events_main(events_args.filename, events_args.verbose);
+        return events_main(events_args.filename, events_args.verbose, events_args.no_yaml);
     }
     case ASDF_SUBCMD_NONE:
         break;