Merge pull request #3 from open-ch/feature/quote_cq_names

quote cq names

Author: Konrad Bucheli

influxdb-schema-updater

@@ -146,10 +146,10 @@ sub extract_database_updates {
     my ($db_schemas_in_influxdb, $db_schemas_in_config, $dryrun, $force) = @_;
 
     my ($old_dbs, $eq_dbs, $new_dbs) = get_Ldifference_intersection_Rdifference([keys %{$db_schemas_in_influxdb}], [keys %{$db_schemas_in_config}]);
-    
+
     my %rp_updates;
     for my $db (@$eq_dbs, @$new_dbs) {
-        my ($old, $updated, $new) = extract_retention_policy_updates($db, $db_schemas_in_influxdb->{$db}, $db_schemas_in_config->{$db}->{rps}, $dryrun, $force);
+        my ($old, $updated, $new) = extract_retention_policy_updates($db, $db_schemas_in_influxdb->{$db}, $db_schemas_in_config->{$db}, $dryrun, $force);
         $rp_updates{old_rps}->{$db} = $old;
         $rp_updates{updated_rps}->{$db} = $updated;
         $rp_updates{new_rps}->{$db} = $new;
@@ -169,7 +169,7 @@ sub extract_database_updates {
             object => 'db',
             db => $db,
             name => $db,
-            query => "DROP DATABASE $db;",
+            query => qq{DROP DATABASE "$db";},
             skip => $dryrun || !$force,
         };
     }
@@ -180,7 +180,7 @@ sub extract_database_updates {
             object => 'db',
             db => $db,
             name => $db,
-            query => $db_schemas_in_config->{$db}->{create_query},
+            query => qq{CREATE DATABASE "$db";},
             skip => $dryrun,
         };
     }
@@ -209,7 +209,7 @@ sub extract_retention_policy_updates {
             object => 'rp',
             db => $db,
             name => $rp,
-            query => "DROP RETENTION POLICY \"$rp\" ON $db;",
+            query => qq{DROP RETENTION POLICY "$rp" ON "$db";},
             skip => $dryrun || !$force,
         };
     }
@@ -221,7 +221,7 @@ sub extract_retention_policy_updates {
                 object => 'rp',
                 db => $db,
                 name => $rp,
-                query => "ALTER RETENTION POLICY \"$rp\" ON $db DURATION $rps_in_config->{$rp}->{duration} REPLICATION 1 SHARD DURATION $rps_in_config->{$rp}->{shard_duration}" . ($rps_in_config->{$rp}->{default} ? ' DEFAULT;' : ';'),
+                query => qq{ALTER RETENTION POLICY "$rp" ON "$db" DURATION $rps_in_config->{$rp}->{duration} REPLICATION 1 SHARD DURATION $rps_in_config->{$rp}->{shard_duration}} . ($rps_in_config->{$rp}->{default} ? ' DEFAULT;' : ';'),
                 skip => $dryrun,
             };
         }
@@ -233,7 +233,7 @@ sub extract_retention_policy_updates {
             object => 'rp',
             db => $db,
             name => $rp,
-            query => "CREATE RETENTION POLICY \"$rp\" ON $db DURATION $rps_in_config->{$rp}->{duration} REPLICATION $rps_in_config->{$rp}->{rp_replication} SHARD DURATION $rps_in_config->{$rp}->{shard_duration}" . ($rps_in_config->{$rp}->{default} ? ' DEFAULT;' : ';'),
+            query => qq{CREATE RETENTION POLICY "$rp" ON "$db" DURATION $rps_in_config->{$rp}->{duration} REPLICATION $rps_in_config->{$rp}->{rp_replication} SHARD DURATION $rps_in_config->{$rp}->{shard_duration}} . ($rps_in_config->{$rp}->{default} ? ' DEFAULT;' : ';'),
             skip => $dryrun,
         };
     }
@@ -276,7 +276,7 @@ sub extract_continuous_query_updates {
                 object => 'cq',
                 db => $db,
                 name => $cq,
-                query => "DROP CONTINUOUS QUERY $cq ON $db;",
+                query => qq{DROP CONTINUOUS QUERY "$cq" ON "$db";},
                 skip => $dryrun || !$force,
             };
         }
@@ -287,7 +287,7 @@ sub extract_continuous_query_updates {
                     object => 'cq',
                     db => $db,
                     name => $cq,
-                    query => "DROP CONTINUOUS QUERY $cq ON $db; $all_cqs_in_config->{$db}->{$cq};",
+                    query => qq{DROP CONTINUOUS QUERY "$cq" ON "$db"; $all_cqs_in_config->{$db}->{$cq};},
                     skip => $dryrun,
                 };
             }
@@ -346,7 +346,7 @@ sub load_db_schemas_in_influxdb {
 
     my %db_schemas_in_influxdb;
     for my $db (@dbs_in_influxdb) {
-        my $rp_query_res = query_influxql($influxdb_client, "SHOW RETENTION POLICIES ON $db");
+        my $rp_query_res = query_influxql($influxdb_client, qq{SHOW RETENTION POLICIES ON "$db"});
         $db_schemas_in_influxdb{$db} = {
             map  { $_->[0] => {
                     duration => $_->[1],
@@ -369,13 +369,11 @@ sub load_db_schemas_in_influxdb {
 #     $schema_dir string: the directory name where the config files are
 #
 # Returns:
-#     a reference to a hash holding the parsed statements. This hash is structured as below.
+#     $db_schemas: a reference to a hash holding the parsed statements. This hash is structured as below.
 #
 # {
 #     <db> => {
-#         create_query => '...',
-#         rps => {
-#             <rp> => {
+#         <rp> => {
 #                 duration => ...,
 #                 shard_duration => ...,
 #                 default => ...,
@@ -389,21 +387,39 @@ sub load_db_schemas_in_influxdb {
 sub load_db_schemas_in_config {
     my ($schema_dir) = @_;
 
-    # this will be populated with all statements parsed from the files
-    my %parsed_statements;
+    my %db_schemas;
     my $db_files = get_schema_files_for_dir("$schema_dir/db");
-    
+
     for my $db_file (@$db_files) {
         my $statements_in_file = read_text("$schema_dir/db/$db_file");
-        my $parsed_statements_from_file = parse_statements($statements_in_file);
-        
-        # add the statements from this file to the initial hash
-        %parsed_statements = (%parsed_statements, %$parsed_statements_from_file);
+        my ($databases, $rps) = parse_statements($statements_in_file);
+
+        # loop all the databases and add them to the hash
+        for my $db (@$databases) {
+            # make sure every database is only created once
+            if (exists $db_schemas{$db}) {
+                die "duplicate database $db in file $db_file detected\n";
+            }
+            $db_schemas{$db} = {};
+        }
+        # loop all the retention policies and assign them to the correct database
+        for my $rp (@$rps) {
+            my $db = $rp->{database};
+            if (exists $db_schemas{$db}) {
+                # make sure every retention policy is only created once
+                if (exists $db_schemas{$db}->{$rp->{rp_name}}) {
+                    die "duplicate rp $rp on db $db in file $db_file detected\n";
+                }
+                $db_schemas{$db}->{$rp->{rp_name}} = $rp;
+            }
+            else {
+                die "database $db specified in rp $rp from file $db_file does not exist\n";
+            }
+        }
     }
-    return \%parsed_statements;
+    return \%db_schemas;
 }
 
-
 #
 # Iterates over the lines of a config file, parses the valid statements and adds them to the given hash.
 # Valid statements are:
@@ -412,19 +428,20 @@ sub load_db_schemas_in_config {
 #
 # Arguments:
 #     $string_to_parse string: the contents of the config file (loaded as string)
-#     $parsed_statements reference: a reference to a hash. The function will populate it with the parsed statements.
 #
 # Returns:
-#     a reference to a hash holding the parsed statements from the file.
+#     $databases: reference to a list which contains all the database names
+#     $rps: reference to a list which contains hashes of all the retention policies
 #
 sub parse_statements {
     my ($string_to_parse) = @_;
     # we want to iterate line-by-line
     my @splitted_lines = split "\n", $string_to_parse;
-    my %parsed_statements;
+    my @databases;
+    my @rps;
 
     # captures a 'create database' statement with an optional retention policy definition
-    my $db_regex = qr/^\s*(create \s+ database \s+ (\w+))             # a create db statement, optionally starting with a space. Any valid word as db name. Captured group.
+    my $db_regex = qr/^\s*(create \s+ database \s+ "?([\w.]+)"?)             # a create db statement, optionally starting with a space. Any valid word as db name. Captured group.
                      (
                          \s+ with \s+ duration                        # optional statement to define a retention policy in the db creation statement
                          \s+ ((?:\d+[smhdw])|(?:inf))                 # captured policy duration as one or more numbers followed by a letter, or 'inf'
@@ -453,7 +470,7 @@ sub parse_statements {
     for my $line (@splitted_lines) {
         # ignore empty and commented lines
         next if $line =~ /^\s*(#.*)?$/;
-        
+
         if ($line =~ /$integrated_regex/) {
             # capture groups from the regex
             my $create_db_statement = $1;
@@ -472,23 +489,30 @@ sub parse_statements {
 
             # the line is a 'create database' statement...
             if ($create_db_statement) {
-                check_if_db_statement_exists_and_die(\%parsed_statements, $db_name);
-                $parsed_statements{$db_name}{create_query} = $create_db_statement . ';';
+                push @databases, $db_name;
                 # ...and has a retention policy defined inline
                 if ($inline_rp_statement) {
-                    # flagged as the default policy (assume that the inline policy is the default)
-                    $parsed_statements{$db_name}{rps}{$inline_rp_name} = {duration       => $inline_rp_duration,
-                                                                          shard_duration => $inline_rp_shard_duration,
-                                                                          rp_replication => $inline_rp_replication,
-                                                                          default        => 1};
+                    # flagged as the default policy (assume that the inline policy is the default)ƒ
+                    push @rps, {
+                        database       => $db_name,
+                        rp_name        => $inline_rp_name,
+                        duration       => $inline_rp_duration,
+                        shard_duration => $inline_rp_shard_duration,
+                        rp_replication => $inline_rp_replication,
+                        default        => 1,
+                    };
                 }
             }
             # the line is a 'create retention policy' statement
             elsif ($rp_name) {
-                $parsed_statements{$rp_db_name}{rps}{$rp_name} = {duration       => $rp_duration,
-                                                                  shard_duration => $rp_shard_duration,
-                                                                  rp_replication => $rp_replication,
-                                                                  default        => $default_rp}
+                push @rps, {
+                    database       => $rp_db_name,
+                    rp_name        => $rp_name,
+                    duration       => $rp_duration,
+                    shard_duration => $rp_shard_duration,
+                    rp_replication => $rp_replication,
+                    default        => $default_rp,
+                }
             }
             else {
                 die "error, should never reach this"
@@ -498,29 +522,9 @@ sub parse_statements {
             die "could not parse input: $line";
         }
     }
-    return \%parsed_statements;
+    return return \@databases, \@rps;
 }
 
-
-#
-# Checks if the db creation statement is already added in the parsed statements hash. If yes then it throws an exception.
-#
-#  Arguments:
-#      $parsed_statements reference: a reference to the hash containing the db statements
-#      $db_name string: 
-#  Returns:
-#      -
-#
-sub check_if_db_statement_exists_and_die {
-    my ($parsed_statements, $db_name) = @_;
-    
-    # there should be no 'create database' statement yet for this database
-    if ($parsed_statements->{$db_name}{create_query}) {
-        die "Create statement already defined for database $db_name. Fix the config file";
-    }
-}
-
-
 # {
 #     <db> => {
 #         <cq_name> => <cq_create_query>,

t/data/test_name_with_dot/cq/db1.ifql

@@ -0,0 +1,2 @@
+CREATE CONTINUOUS QUERY "cq.test" ON "db.test" RESAMPLE EVERY 5m FOR 10m BEGIN SELECT LAST(a) AS b, c INTO "rp.test"."m.test" FROM "rp.test"."m.test" GROUP BY time(5m) END;
+

t/data/test_name_with_dot/db/db1.ifql

@@ -0,0 +1 @@
+CREATE DATABASE "db.test" WITH DURATION 260w REPLICATION 1 SHARD DURATION 12w NAME "rp.test";