Bug fix, BUFR to BUFR (#534)

* bufr to bufr bug fix, headers now copied and set in the output BUFR files.

* Correctly set the number of subsets in output of bufr to bufr.

* Flake8, sigh.

* Update tests-docker.yml

---------

Co-authored-by: Maaike <maaike.limper@gmail.com>

Author: david-i-berry

.github/workflows/tests-docker.yml

@@ -109,18 +109,17 @@ jobs:
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: sync stations 🔄
       run: |
-        sleep 15
+        sleep 30
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection
     - name: run integration tests ⚙️
       run: |
-        sleep 15
+        sleep 30
         pytest -s tests/integration
     - name: run flake8 ⚙️
       run: |
         find . -type f -name "*.py" | xargs flake8
     - name: failed tests 🚩
       if: ${{ failure() }}
       run: |
-        docker logs wis2box-management
-        docker logs wis2box-api
-        docker logs wis2box-minio
+        docker logs -n1000 wis2box-management
+        docker logs -n1000 wis2box-api

wis2box-management/wis2box/data/bufr4.py

@@ -54,9 +54,18 @@
     'typicalSecond'
 ]
 
+HEADERS = ["edition", "masterTableNumber", "bufrHeaderCentre",
+           "bufrHeaderSubCentre", "updateSequenceNumber", "dataCategory",
+           "internationalDataSubCategory", "dataSubCategory",
+           "masterTablesVersionNumber", "localTablesVersionNumber",
+           "typicalYear", "typicalMonth", "typicalDay", "typicalHour",
+           "typicalMinute", "typicalSecond",
+           "numberOfSubsets", "observedData", "compressedData",
+           "unexpandedDescriptors"]
+
 
 class ObservationDataBUFR(BaseAbstractData):
-    """Oservation data"""
+    """Observation data"""
 
     def transform(
         self, input_data: Union[Path, bytes], filename: str = ''
@@ -97,6 +106,13 @@ def transform_message(self, bufr_in: int) -> None:
             LOGGER.error(f'Error unpacking message: {err}')
             raise err
 
+        # get descriptors present in the file
+        descriptors = codes_get_array(bufr_in, "expandedDescriptors").tolist()
+        # get the headers in hte file
+        headers = {}
+        for header in HEADERS:
+            headers[header] = codes_get(bufr_in, header)
+
         num_subsets = codes_get(bufr_in, 'numberOfSubsets')
         LOGGER.debug(f'Found {num_subsets} subsets')
 
@@ -112,15 +128,41 @@ def transform_message(self, bufr_in: int) -> None:
             idx = i + 1
             LOGGER.debug(f'Processing subset {idx}')
 
-            LOGGER.debug('Copying template BUFR')
-            subset_out = codes_clone(TEMPLATE)
-            codes_set(subset_out, 'masterTablesVersionNumber', table_version)
-            codes_set_array(subset_out, 'unexpandedDescriptors', outUE)
-
             LOGGER.debug('Extracting subset')
             codes_set(bufr_in, 'extractSubset', idx)
             codes_set(bufr_in, 'doExtractSubsets', 1)
 
+            # copy the replication factors
+            if 31000 in descriptors:
+                short_replication_factors = codes_get_array(bufr_in, "shortDelayedDescriptorReplicationFactor").tolist()  # noqa
+            if 31001 in descriptors:
+                replication_factors = codes_get_array(bufr_in, "delayedDescriptorReplicationFactor").tolist()  # noqa
+            if 31002 in descriptors:
+                extended_replication_factors = codes_get_array(bufr_in, "extendedDelayedDescriptorReplicationFactor").tolist()  # noqa
+
+            LOGGER.debug('Copying template BUFR')
+            subset_out = codes_clone(TEMPLATE)
+
+            # set the replication factors, this needs to be done before
+            # setting the unexpanded descriptors
+            if 31000 in descriptors:
+                codes_set_array(subset_out, "inputShortDelayedDescriptorReplicationFactor", short_replication_factors)  # noqa
+            if 31001 in descriptors:
+                codes_set_array(subset_out, "inputDelayedDescriptorReplicationFactor", replication_factors)  # noqa
+            if 31002 in descriptors:
+                codes_set_array(subset_out, "inputExtendedDelayedDescriptorReplicationFactor", extended_replication_factors)  # noqa
+
+            # we need to copy all the headers, not just the
+            # unexpandedDescriptors and MT number
+            headers['unexpandedDescriptors'] = outUE
+            headers['masterTablesVersionNumber'] = table_version
+            headers['numberOfSubsets'] = 1
+            for k, v in headers.items():
+                if isinstance(v, list):
+                    codes_set_array(subset_out, k, v)
+                else:
+                    codes_set(subset_out, k, v)
+
             LOGGER.debug('Cloning subset to new message')
             subset = codes_clone(bufr_in)
             self.transform_subset(subset, subset_out)