diff --git a/.gitignore b/.gitignore index e01f0e10d..cd3199732 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ target/ /ingestion-tool/src/temp/ temp.txt +temp.java +temp.xml /out/ \ No newline at end of file diff --git a/CHANGES.md b/CHANGES.md index 65bacf9ea..a895a379c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,36 @@ +### Updates from version 1.5.8 to 1.6.0 +* Migrated to SNAP 11, several refactorings and library updates +* The VariableCache now uses an LRU cache with a listener to automatically free the oldest cached, no longer + needed variables and file handles when the maximum cache size is reached. +* fix wrong corner point order of era5 interpolation in SatelliteFields. + Instead, the four era5 interpolation vertices are read directly from the variable +* Remove reading of subset from entire era5 variable array and remove to read the entire era5 variable in VariableCache +* Remove creation of rectangles for InterpolationContext and remove field era5Region in InterpolationContext +* era5 post processing .. can now also handle satellite longitude data which not fits the range [-180 to 180]. + In such cases (e.g. Windsat-Coriolis [0 to 360]), longitude data will be converted so that it fits into the + required range of [-180 to 180] to be able to create correct interpolation of era5 data. +* era5 post processing .. In era5-post-processing-general-info.xml a fill value per + variable can be defined. +* era5 post processing .. can now load generalized information from an optional + era5-post-processing-general-info.xml in the config directory. + A general info file can contain variable definition for satellite-fields. +* Upgrade Mockito to version 4.11.0 and also include artifact mockito-inline for static mocking +* era5 post processing .. setter and getter of variable names in class SatelliteFieldsConfiguration was + generalized. E.g. config.set_an_ml_q_name("abcde") was replaced by config.setVarName("an_ml_q", "abcde") +* era5 post processing .. "length" attribute of tag " is no + longer optional. +* era5 post processing .. The translation of variable name to variable name for file access has been made + switchable. +* post processing .. added PostProcessingContext to method PostProcessingPlugin.createPostProcessing(...). + Now plugins can perform extended initializations. E.g. by loading extra files from config directory. +* post processing .. the config directory has been added to the PostProcessingContext so that plugins can load + additional generalized information required for the plugin from this directory. + ### Updates from version 1.5.7 to 1.5.8 +* added support for SMAP Salinity L2C data +* added functionality to configure several global attributes before computing MMD files. +* added support for NOAA TAO insitu data +* added support for NOAA NDBC insitu data * added support for Windsat Coriolis data * added conda environment * extended workflow to run without scheduler on plain console diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..00205d227 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,37 @@ +cff-version: 1.2.0 +title: Multisensor Matchup System (MMS) +message: >- + If you use this software, please cite it using the + metadata from this file. +type: software +authors: + - given-names: Thomas + family-names: Block + email: tom.block@brockmann-consult.de + affiliation: Brockmann Consult GmbH + - given-names: Sabine + family-names: Embacher + email: sabine.embacher@brockmann-consult.de + affiliation: Brockmann Consult GmbH +identifiers: + - type: doi + value: 10.5281/zenodo.8116892 +repository-code: 'https://github.com/bcdev/MMS' +url: 'https://www.brockmann-consult.de/' +abstract: >- + The Multisensor matchup System (MMS) is a software suite + that allows to detect simulatneous measurements of remote + sensing (satellite) and in-situ instruments. + + + Its main purpose is to support the validation of satellite + based instruments by generating cross-sensor matches of + either instruments from different satellite platforms or + between satellite and ground-based measurements. +keywords: + - matchup + - calibration + - validation + - earth observation + - satellite +license: GPL-3.0 diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 000000000..94a9ed024 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/MMS_Manual_v_1_5.pdf b/MMS_Manual_v_1_5.pdf deleted file mode 100644 index 8db1e15bf..000000000 Binary files a/MMS_Manual_v_1_5.pdf and /dev/null differ diff --git a/MMS_Manual_v_1_6.pdf b/MMS_Manual_v_1_6.pdf new file mode 100644 index 000000000..d99900cd8 Binary files /dev/null and b/MMS_Manual_v_1_6.pdf differ diff --git a/build_and_install_on_cems.sh b/build_and_install_on_cems.sh index 5ad00b845..3e71c50fe 100755 --- a/build_and_install_on_cems.sh +++ b/build_and_install_on_cems.sh @@ -6,5 +6,5 @@ mvn clean install package assembly:directory # echo "clean up bin dir" echo "copy build result to bin dir" rm -rf /gws/nopw/j04/fiduceo/Software/mms/bin/* -cp -a target/fiduceo-master-1.5.8-MMS/* /gws/nopw/j04/fiduceo/Software/mms/bin +cp -a target/fiduceo-master-1.6.2-MMS/* /gws/nopw/j04/fiduceo/Software/mms/bin diff --git a/build_and_install_on_cems_sst.sh b/build_and_install_on_cems_sst.sh index bfb4e7b1c..94890f6b8 100755 --- a/build_and_install_on_cems_sst.sh +++ b/build_and_install_on_cems_sst.sh @@ -6,5 +6,5 @@ mvn clean install package assembly:directory # echo "clean up bin dir" echo "copy build result to bin dir" rm -rf /gws/nopw/j04/esacci_sst/mms_new/bin/* -cp -a target/fiduceo-master-1.5.8-MMS/* /gws/nopw/j04/esacci_sst/mms_new/bin +cp -a target/fiduceo-master-1.6.2-MMS/* /gws/nopw/j04/esacci_sst/mms_new/bin diff --git a/cems/pom.xml b/cems/pom.xml index 82da1d8fa..887eee1ad 100644 --- a/cems/pom.xml +++ b/cems/pom.xml @@ -25,7 +25,7 @@ fiduceo-master com.bc.fiduceo - 1.5.8 + 1.6.2 4.0.0 diff --git a/cems/src/main/bin/mms-env.sh b/cems/src/main/bin/mms-env.sh index 88f5c6445..d2faa76a2 100755 --- a/cems/src/main/bin/mms-env.sh +++ b/cems/src/main/bin/mms-env.sh @@ -8,13 +8,14 @@ # project and user settings # ------------------------- -export PROJECT=bc_fiduceo # only LSF -export MMS_USER=tblock01 # only SLURM +export PROJECT=bc_fiduceo # only LSF +export MMS_USER=tblock01 # only SLURM +export MMS_ACCOUNT=esacci_sst # only SLURM_2 export MMS_ENV_NAME=mms-env.sh # Java and Python runtime definitions # ----------------------------------- -export MMS_JAVA_EXEC='/gws/nopw/j04/esacci_sst/mms_new/software/jdk1.8.0_202/bin/java' +export MMS_JAVA_EXEC='/gws/nopw/j04/esacci_sst/mms_new/software/openjdk-11.0.26' export PM_EXE_DIR=/gws/nopw/j04/esacci_sst/mms_new/bin export PM_PYTHON_EXEC='/gws/nopw/j04/esacci_sst/mms_new/software/conda_envs/sst-cci-mms/bin/python' @@ -23,7 +24,8 @@ export PATH=${PM_EXE_DIR}:$PATH # export scheduling engine # ------------------------ -export SCHEDULER='SLURM' +export SCHEDULER='SLURM_2' +# export SCHEDULER='SLURM' # export SCHEDULER='LSF' # ensure that processes exit @@ -35,6 +37,9 @@ fi export PM_LOG_DIR=${WORKING_DIR}/log +# -------------------------------------- +# --- LSF ------------------------------ +# -------------------------------------- if [ "$SCHEDULER" == "LSF" ]; then submit_job() { @@ -61,6 +66,9 @@ if [ "$SCHEDULER" == "LSF" ]; then fi } +# -------------------------------------- +# --- SLURM ---------------------------- +# -------------------------------------- elif [ "$SCHEDULER" == "SLURM" ]; then submit_job() { @@ -87,6 +95,35 @@ elif [ "$SCHEDULER" == "SLURM" ]; then fi } +# -------------------------------------- +# --- SLURM_2 -------------------------- +# -------------------------------------- +elif [ "$SCHEDULER" == "SLURM_2" ]; then + +submit_job() { + jobname=$1 + command=$2 + + bsubmit="sbatch -A ${MMS_ACCOUNT} --mem=20000 -p standard -q short -n 1 -t 04:00:00 -o ${PM_LOG_DIR}/${jobname}.out -e ${PM_LOG_DIR}/${jobname}.err --job-name ${jobname} ${PM_EXE_DIR}/${command} ${@:3}" + + rm -f ${PM_LOG_DIR}/${jobname}.out + rm -f ${PM_LOG_DIR}/${jobname}.err + + # line contains the console output of the bsub command + line=`${bsubmit}` + + if echo ${line} | grep -qF 'Submitted batch job' + then + # extract the job_id from the bsub message, concatenate '_' and jobname to form an identifier + # and dump to std_out to be fetched by pmonitor + job_id=`echo ${line} | awk '{ print substr($4,0,length($4)) }'` + echo "${job_id}_${jobname}" + else + echo "`date -u +%Y%m%d-%H%M%S` - submit of ${jobname} failed: ${line}" + exit 1 + fi + } + else echo "Invalid scheduler" exit 1 diff --git a/cems/src/main/bin/post_processing_run.sh b/cems/src/main/bin/post_processing_run.sh index 821651933..8a536bc0f 100644 --- a/cems/src/main/bin/post_processing_run.sh +++ b/cems/src/main/bin/post_processing_run.sh @@ -8,6 +8,6 @@ end_date=$3 job_config=$4 config_dir=$5 -echo "`date -u +%Y%m%d-%H%M%S` ingestion ${start_date} - ${end_date} ..." +echo "`date -u +%Y%m%d-%H%M%S` post_processing ${start_date} - ${end_date} ..." ${PM_EXE_DIR}/post-processing-tool.sh -i ${input_dir} -start ${start_date} -end ${end_date} -j ${job_config} -c ${config_dir} diff --git a/cems/src/main/python/jasmin/jasmin_job_monitor.py b/cems/src/main/python/jasmin/jasmin_job_monitor.py index 12675208e..2d3f90230 100644 --- a/cems/src/main/python/jasmin/jasmin_job_monitor.py +++ b/cems/src/main/python/jasmin/jasmin_job_monitor.py @@ -48,8 +48,10 @@ def _create_scheduler_interface(): return LSFInterface() elif scheduler_name == "SLURM": return SLURMInterface() + elif scheduler_name == "SLURM_2": + return SLURMInterface() else: - raise ValueError("Environment variable 'SCHEDULER' invalid") + raise ValueError("Environment variable 'SCHEDULER' invalid" + scheduler_name) else: raise ValueError("Environment variable 'SCHEDULER' is not set") @@ -188,10 +190,13 @@ def resolve_status_from_log(self, check_log_dict): class SLURMInterface: user_name = None + scheduler_name = None def __init__(self): if "MMS_USER" in os.environ: self.user_name = os.environ["MMS_USER"] + if "SCHEDULER" in os.environ: + self.scheduler_name = os.environ["SCHEDULER"] else: raise RuntimeError("Missing environment variable 'MMS_USER'") @@ -224,7 +229,12 @@ def _extract_id_and_status(self, line): if len(tokens) < 7: raise ValueError("unable to handle 'squeue' result: " + line) - status_code = self._status_to_enum(tokens[4]) + if self.scheduler_name == "SLURM_2": + status_token = tokens[7] + else: + status_token = tokens[4] + + status_code = self._status_to_enum(status_token) return {tokens[0]: status_code} @staticmethod diff --git a/cems/src/main/python/jasmin/jasmin_job_monitor_test.py b/cems/src/main/python/jasmin/jasmin_job_monitor_test.py index adcc7b202..4c4f2b7bb 100644 --- a/cems/src/main/python/jasmin/jasmin_job_monitor_test.py +++ b/cems/src/main/python/jasmin/jasmin_job_monitor_test.py @@ -2,8 +2,8 @@ import io import unittest -from jasmin.jasmin_job_monitor import JasminJobMonitor, LSFInterface, SLURMInterface -from jasmin.status_codes import StatusCodes +from jasmin_job_monitor import JasminJobMonitor, LSFInterface, SLURMInterface +from status_codes import StatusCodes class JasminJobMonitorTest(unittest.TestCase): @@ -62,6 +62,25 @@ def test_parse_job_status_SLURM(self): finally: del os.environ["MMS_USER"] + def test_parse_job_status_SLURM_2(self): + output = " JOBID PARTIT QOS NAME USER NODE CPUS ST TIME TIME_LEFT PRIORITY NODELIST(REASON)\n" \ + "8466477 standa short ingest-slstr-s3a-uor tblock01 1 1 R 0:37 3:59:23 77911 (NonZeroExitCode)\n" \ + "8466478 standa short ingest-slstr-s3a-uor tblock01 1 1 F 0:37 3:59:23 77911 (NonZeroExitCode)" + + + try: + os.environ["MMS_USER"] = "HarryPotter" + os.environ["SCHEDULER"] = "SLURM_2" + + scheduler = SLURMInterface() + job_status_dict = scheduler.parse_jobs_call(output) + self.assertEqual(2, len(job_status_dict)) + self.assertEqual(StatusCodes.RUNNING, job_status_dict["8466477"]) + self.assertEqual(StatusCodes.FAILED, job_status_dict["8466478"]) + finally: + del os.environ["MMS_USER"] + del os.environ["SCHEDULER"] + def test_status_to_enum_LSF(self): scheduler = LSFInterface() diff --git a/cems/src/main/python/workflow.py b/cems/src/main/python/workflow.py index 3552ab219..09ca9571d 100644 --- a/cems/src/main/python/workflow.py +++ b/cems/src/main/python/workflow.py @@ -408,16 +408,23 @@ def run_matchup(self, hosts, num_parallel_tasks, simulation=False, logdir='trace monitor.wait_for_completion() - def run_post_processing(self, hosts, num_parallel_tasks, simulation=False, logdir='trace'): + def run_post_processing(self, hosts, num_parallel_tasks, simulation=False, logdir='trace', synchronous=False): """ - :param hosts: list :param num_parallel_tasks: int :param simulation: bool :param logdir: str + :param synchronous: bool: + decides whether post_processing_run.sh (True) or post_processing_start.sh (False) is called. :return: """ - monitor = self._get_monitor(hosts, [('post_processing_start.sh', num_parallel_tasks)], logdir, simulation) + + if synchronous: + runs_script = 'post_processing_run.sh' + else: + runs_script = 'post_processing_start.sh' + + monitor = self._get_monitor(hosts, [(runs_script, num_parallel_tasks)], logdir, simulation, synchronous) production_period = self.get_production_period() date = production_period.get_start_date() while date < production_period.get_end_date(): @@ -430,7 +437,7 @@ def run_post_processing(self, hosts, num_parallel_tasks, simulation=False, logdi pre_condition = 'mmd-' + start_string + '-' + end_string post_condition = 'post-processing-' + start_string + '-' + end_string + '-' + self.usecase_config - job = Job(job_name, 'post_processing_start.sh', [pre_condition], [post_condition], + job = Job(job_name, runs_script, [pre_condition], [post_condition], [self.input_dir, start_string, end_string, self.usecase_config, self._get_config_dir()]) monitor.execute(job) diff --git a/core/pom.xml b/core/pom.xml index 09322a5be..4ff871bb5 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -5,7 +5,7 @@ fiduceo-master com.bc.fiduceo - 1.5.8 + 1.6.2 4.0.0 @@ -39,13 +39,13 @@ - org.esa.s3tbx - s3tbx-avhrr-reader + eu.esa.opt + opttbx-avhrr-reader - org.esa.s3tbx - s3tbx-sentinel3-reader + eu.esa.opt + opttbx-sentinel3-reader @@ -89,14 +89,10 @@ netcdfAll - - - - - - - - + + org.jdom + jdom2 + junit @@ -110,7 +106,7 @@ org.hamcrest - hamcrest-all + hamcrest test @@ -118,6 +114,12 @@ jimfs test + + org.apache.commons + commons-text + 1.10.0 + test + diff --git a/core/src/main/bin/mmd_qc_tool.bat b/core/src/main/bin/mmd_qc_tool.bat new file mode 100644 index 000000000..e51e105c7 --- /dev/null +++ b/core/src/main/bin/mmd_qc_tool.bat @@ -0,0 +1,30 @@ +@ECHO OFF + +:: -======================- +:: User configurable values +:: -======================- + +SET INSTALLDIR=%~dp0% + +::------------------------------------------------------------------ +:: You can adjust the Java minimum and maximum heap space here. +:: Just change the Xms and Xmx options. Space is given in megabyte. +:: '-Xms64M' sets the minimum heap space to 64 megabytes +:: '-Xmx512M' sets the maximum heap space to 512 megabytes +::------------------------------------------------------------------ +SET JAVA_OPTS=-Xms64M -Xmx8192M + + +:: -======================- +:: Other values +:: -======================- + +SET JAVAEXE=java.exe +SET LIBDIR=%INSTALLDIR%\lib +SET OLD_CLASSPATH=%CLASSPATH% + +SET CLASSPATH=%LIBDIR%\*;%LIBDIR% + +CALL "%JAVAEXE%" %JAVA_OPTS% -classpath "%CLASSPATH%" com.bc.fiduceo.qc.MmdQCToolMain %* + +SET CLASSPATH=%OLD_CLASSPATH% diff --git a/core/src/main/bin/mmd_qc_tool.sh b/core/src/main/bin/mmd_qc_tool.sh new file mode 100644 index 000000000..6cb78d379 --- /dev/null +++ b/core/src/main/bin/mmd_qc_tool.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# -======================- +# User configurable values +# -======================- + +export INSTALLDIR="$(dirname $0)" + +#------------------------------------------------------------------ +# You can adjust the Java minimum and maximum heap space here. +# Just change the Xms and Xmx options. Space is given in megabyte. +# '-Xms64M' sets the minimum heap space to 64 megabytes +# '-Xmx512M' sets the maximum heap space to 512 megabytes +#------------------------------------------------------------------ +export JAVA_OPTS="-Xmx8192M" + +# check if we`re running on CEMS, if so take the java executable externally defined +if [ -z "${MMS_JAVA_EXEC}" ]; then + # not set, use what we have on the system + export JAVA_EXE="$(which java)" +else + # yes, we seem to be on CEMS + export JAVA_EXE="$MMS_JAVA_EXEC" +fi + + +# -======================- +# Other values +# -======================- + +export LIBDIR="$INSTALLDIR"/lib +export OLD_CLASSPATH="$CLASSPATH" +CLASSPATH="$LIBDIR/*:$LIBDIR" + +"$JAVA_EXE" "$JAVA_OPTS" -classpath "$CLASSPATH" com.bc.fiduceo.qc.MmdQCToolMain "$@" + +export CLASSPATH="$OLD_CLASSPATH" diff --git a/core/src/main/java/com/bc/fiduceo/archive/ArchiveConfig.java b/core/src/main/java/com/bc/fiduceo/archive/ArchiveConfig.java index 454ba90bf..dba7742fc 100644 --- a/core/src/main/java/com/bc/fiduceo/archive/ArchiveConfig.java +++ b/core/src/main/java/com/bc/fiduceo/archive/ArchiveConfig.java @@ -42,11 +42,11 @@ import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Attribute; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.JDOMException; -import org.jdom.input.SAXBuilder; +import org.jdom2.Attribute; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.JDOMException; +import org.jdom2.input.SAXBuilder; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/core/src/main/java/com/bc/fiduceo/core/GeoRect.java b/core/src/main/java/com/bc/fiduceo/core/GeoRect.java deleted file mode 100644 index 9f032aea2..000000000 --- a/core/src/main/java/com/bc/fiduceo/core/GeoRect.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.bc.fiduceo.core; - -public class GeoRect { - - private final float lonMin; - private final float lonMax; - private final float latMin; - private final float latMax; - - public GeoRect(float lonMin, float lonMax, float latMin, float latMax) { - this.lonMin = lonMin; - this.lonMax = lonMax; - this.latMin = latMin; - this.latMax = latMax; - } - - public float getLonMin() { - return lonMin; - } - - public float getLonMax() { - return lonMax; - } - - public float getLatMin() { - return latMin; - } - - public float getLatMax() { - return latMax; - } -} diff --git a/core/src/main/java/com/bc/fiduceo/core/IntRange.java b/core/src/main/java/com/bc/fiduceo/core/IntRange.java new file mode 100644 index 000000000..8b15dbbae --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/core/IntRange.java @@ -0,0 +1,40 @@ +package com.bc.fiduceo.core; + +public class IntRange { + + private int min; + private int max; + + public IntRange() { + this(Integer.MAX_VALUE, Integer.MIN_VALUE); + } + + public IntRange(int min, int max) { + this.min = min; + this.max = max; + } + + public int getMin() { + return min; + } + + public int getMax() { + return max; + } + + public void setMin(int min) { + this.min = min; + } + + public void setMax(int max) { + this.max = max; + } + + public int getLength() { + return max - min + 1; + } + + public boolean contains(int value) { + return value >= min && value <= max; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/core/SystemConfig.java b/core/src/main/java/com/bc/fiduceo/core/SystemConfig.java index 01a10173e..5020e9709 100644 --- a/core/src/main/java/com/bc/fiduceo/core/SystemConfig.java +++ b/core/src/main/java/com/bc/fiduceo/core/SystemConfig.java @@ -22,10 +22,10 @@ import com.bc.fiduceo.archive.ArchiveConfig; import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.JDOMException; -import org.jdom.input.SAXBuilder; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.JDOMException; +import org.jdom2.input.SAXBuilder; import java.io.File; import java.io.FileInputStream; diff --git a/core/src/main/java/com/bc/fiduceo/core/UseCaseConfig.java b/core/src/main/java/com/bc/fiduceo/core/UseCaseConfig.java index 90fe07698..1c37ffdbf 100644 --- a/core/src/main/java/com/bc/fiduceo/core/UseCaseConfig.java +++ b/core/src/main/java/com/bc/fiduceo/core/UseCaseConfig.java @@ -21,12 +21,12 @@ package com.bc.fiduceo.core; import org.esa.snap.core.util.StringUtils; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.JDOMException; -import org.jdom.input.SAXBuilder; -import org.jdom.output.Format; -import org.jdom.output.XMLOutputter; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.JDOMException; +import org.jdom2.input.SAXBuilder; +import org.jdom2.output.Format; +import org.jdom2.output.XMLOutputter; import java.io.IOException; import java.io.InputStream; diff --git a/core/src/main/java/com/bc/fiduceo/db/DbMaintenanceTool.java b/core/src/main/java/com/bc/fiduceo/db/DbMaintenanceTool.java index 54d300842..5fb5ccd82 100644 --- a/core/src/main/java/com/bc/fiduceo/db/DbMaintenanceTool.java +++ b/core/src/main/java/com/bc/fiduceo/db/DbMaintenanceTool.java @@ -8,7 +8,7 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.io.IOException; import java.io.OutputStream; diff --git a/core/src/main/java/com/bc/fiduceo/db/DbMaintenanceToolMain.java b/core/src/main/java/com/bc/fiduceo/db/DbMaintenanceToolMain.java index 2e683c4fb..dbf6ab92d 100644 --- a/core/src/main/java/com/bc/fiduceo/db/DbMaintenanceToolMain.java +++ b/core/src/main/java/com/bc/fiduceo/db/DbMaintenanceToolMain.java @@ -1,9 +1,6 @@ package com.bc.fiduceo.db; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; +import org.apache.commons.cli.*; import java.io.IOException; import java.sql.SQLException; @@ -18,7 +15,7 @@ public static void main(String[] args) throws ParseException { return; } - final CommandLineParser parser = new PosixParser(); + final CommandLineParser parser = new DefaultParser(); final CommandLine commandLine = parser.parse(DbMaintenanceTool.getOptions(), args); if (commandLine.hasOption("h") || commandLine.hasOption("--help")) { maintenanceTool.printUsageTo(System.out); diff --git a/core/src/main/java/com/bc/fiduceo/geometry/GeometryUtil.java b/core/src/main/java/com/bc/fiduceo/geometry/GeometryUtil.java index 54fc796ab..a17dbc345 100644 --- a/core/src/main/java/com/bc/fiduceo/geometry/GeometryUtil.java +++ b/core/src/main/java/com/bc/fiduceo/geometry/GeometryUtil.java @@ -127,6 +127,54 @@ public static String toKml(Polygon polygon) { return builder.toString(); } + public static String toKml(MultiPolygon multiPolygon) { + final StringBuilder builder = new StringBuilder(); + + builder.append("\n"); + builder.append("\n"); + builder.append("\n"); + builder.append(" \n"); + + int index = 1; + for (Polygon polygon : multiPolygon.getPolygons()) { + builder.append(" \n"); + builder.append(" part_" + index + "\n"); + builder.append(" #polygonStyle\n"); + builder.append(" \n"); + builder.append(" clampToGround\n"); + builder.append(" \n"); + builder.append(" \n"); + builder.append(" \n"); + final Point[] coordinates = polygon.getCoordinates(); + for (final Point coordinate : coordinates) { + builder.append(" "); + builder.append(coordinate.getLon()); + builder.append(","); + builder.append(coordinate.getLat()); + builder.append(",0\n"); + } + builder.append(" \n"); + builder.append(" \n"); + builder.append(" \n"); + builder.append(" \n"); + builder.append(" \n"); + index++; + } + + builder.append("\n"); + builder.append(""); + + return builder.toString(); + } + public static String toKml(float[] lats, float[] lons) { final StringBuilder builder = new StringBuilder(); builder.append("\n"); diff --git a/core/src/main/java/com/bc/fiduceo/geometry/Point.java b/core/src/main/java/com/bc/fiduceo/geometry/Point.java index c738630ea..ffd5d2d55 100644 --- a/core/src/main/java/com/bc/fiduceo/geometry/Point.java +++ b/core/src/main/java/com/bc/fiduceo/geometry/Point.java @@ -32,5 +32,5 @@ public interface Point extends Geometry { void setLat(double lat); - boolean equals(Point other); + boolean equals(Object other); } diff --git a/core/src/main/java/com/bc/fiduceo/geometry/jts/JTSPoint.java b/core/src/main/java/com/bc/fiduceo/geometry/jts/JTSPoint.java index 273e48907..965731959 100644 --- a/core/src/main/java/com/bc/fiduceo/geometry/jts/JTSPoint.java +++ b/core/src/main/java/com/bc/fiduceo/geometry/jts/JTSPoint.java @@ -80,7 +80,11 @@ public void setLat(double lat) { } @Override - public boolean equals(Point other) { + public boolean equals(Object o) { + if (o == null || !(o instanceof JTSPoint)) { + return false; + } + final JTSPoint other = (JTSPoint) o; return other == this || other.getLon() == getLon() && other.getLat() == getLat(); } } diff --git a/core/src/main/java/com/bc/fiduceo/geometry/s2/BcS2Point.java b/core/src/main/java/com/bc/fiduceo/geometry/s2/BcS2Point.java index 9d8f0128d..dacfde8a4 100644 --- a/core/src/main/java/com/bc/fiduceo/geometry/s2/BcS2Point.java +++ b/core/src/main/java/com/bc/fiduceo/geometry/s2/BcS2Point.java @@ -103,7 +103,11 @@ public String toString() { } @Override - public boolean equals(Point other) { + public boolean equals(Object o) { + if (o == null || !(o instanceof BcS2Point)) { + return false; + } + final BcS2Point other = (BcS2Point) o; return other == this || other.getLon() == getLon() && other.getLat() == getLat(); } diff --git a/core/src/main/java/com/bc/fiduceo/location/PixelGeoCodingPixelLocator.java b/core/src/main/java/com/bc/fiduceo/location/PixelGeoCodingPixelLocator.java new file mode 100644 index 000000000..89ff2c2fe --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/location/PixelGeoCodingPixelLocator.java @@ -0,0 +1,55 @@ +package com.bc.fiduceo.location; + +import org.esa.snap.core.dataio.geocoding.*; +import org.esa.snap.core.dataio.geocoding.forward.PixelForward; +import org.esa.snap.core.dataio.geocoding.inverse.PixelQuadTreeInverse; +import org.esa.snap.core.datamodel.GeoPos; +import org.esa.snap.core.datamodel.PixelPos; +import ucar.ma2.Array; +import ucar.ma2.DataType; + +import java.awt.geom.Point2D; + +public class PixelGeoCodingPixelLocator implements PixelLocator { + + private final ComponentGeoCoding geoCoding; + private boolean hasHalfPxOffset; + + public PixelGeoCodingPixelLocator(Array longitudes, Array latitudes, String lonVariableName, String latVariableName, double groundResolutionInKm, GeoChecks geoChecks) { + final double[] lonArray = (double[]) longitudes.get1DJavaArray(DataType.DOUBLE); + final double[] latArray = (double[]) latitudes.get1DJavaArray(DataType.DOUBLE); + final int[] shape = longitudes.getShape(); + + final GeoRaster geoRaster = new GeoRaster(lonArray, latArray, lonVariableName, latVariableName, shape[1], shape[0], groundResolutionInKm); + final ForwardCoding forwardCoding = ComponentFactory.getForward(PixelForward.KEY); + final InverseCoding inverseCoding = ComponentFactory.getInverse(PixelQuadTreeInverse.KEY); + geoCoding = new ComponentGeoCoding(geoRaster, forwardCoding, inverseCoding, geoChecks); + geoCoding.initialize(); + + hasHalfPxOffset = true; + } + + public void setHasHalfPxOffset(boolean hasHalfPxOffset) { + this.hasHalfPxOffset = hasHalfPxOffset; + } + + @Override + public Point2D getGeoLocation(double x, double y, Point2D g) { + PixelPos pixelPos; + + if (hasHalfPxOffset) { + pixelPos = new PixelPos(x + 0.5, y + 0.5); + } else { + pixelPos = new PixelPos(x, y); + } + + final GeoPos geoPos = geoCoding.getGeoPos(pixelPos, null); + return new Point2D.Double(geoPos.lon, geoPos.lat); + } + + @Override + public Point2D[] getPixelLocation(double lon, double lat) { + final PixelPos pixelPos = geoCoding.getPixelPos(new GeoPos(lat, lon), null); + return new Point2D[]{new Point2D.Double(pixelPos.getX() - 0.5, pixelPos.getY() - 0.5)}; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/qc/GlobalPlot.java b/core/src/main/java/com/bc/fiduceo/qc/GlobalPlot.java new file mode 100644 index 000000000..f60de287f --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/qc/GlobalPlot.java @@ -0,0 +1,58 @@ +package com.bc.fiduceo.qc; + +import org.esa.snap.core.datamodel.GeoPos; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.*; +import java.util.ArrayList; + +class GlobalPlot { + + static byte[] imageBuffer; + + private BufferedImage image; + + private GlobalPlot() throws IOException { + image = ImageIO.read(new ByteArrayInputStream(imageBuffer)); + } + + static GlobalPlot create() throws IOException { + if (imageBuffer == null) { + final InputStream is = GlobalPlot.class.getResourceAsStream("bluemarble-2048.png"); + if (is == null) { + throw new IllegalStateException("The internal resource file could not be read."); + } + final DataInputStream dis = new DataInputStream(is); + final int fileSize = dis.available(); + imageBuffer = new byte[fileSize]; + dis.readFully(imageBuffer); + } + + return new GlobalPlot(); + } + + void plot(ArrayList pointList) { + final Graphics2D graphics = image.createGraphics(); + graphics.setColor(Color.magenta); + + for (final GeoPos point : pointList) { + final double x = (point.getLon() + 180.0) / 360.0; + final double y = (90.0 - point.getLat()) / 180.0; + final int i = (int) (y * 1024); + final int k = (int) (x * 2048); + graphics.fill(new Rectangle(k, i, 1, 1)); + } + + graphics.dispose(); + } + + void writeTo(String pngFilePath) throws IOException { + ImageIO.write(image, "png", new File(pngFilePath)); + } + + public void dispose() { + image = null; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/qc/MmdQCTool.java b/core/src/main/java/com/bc/fiduceo/qc/MmdQCTool.java index a4c0abd5a..7a4487c44 100644 --- a/core/src/main/java/com/bc/fiduceo/qc/MmdQCTool.java +++ b/core/src/main/java/com/bc/fiduceo/qc/MmdQCTool.java @@ -3,19 +3,20 @@ import com.bc.fiduceo.FiduceoConstants; import com.bc.fiduceo.log.FiduceoLogger; import com.bc.fiduceo.util.NetCDFUtils; +import com.bc.fiduceo.util.TimeUtils; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.esa.snap.core.datamodel.GeoPos; +import org.esa.snap.core.util.StringUtils; +import org.esa.snap.core.util.io.FileUtils; import ucar.ma2.Array; import ucar.ma2.IndexIterator; import ucar.ma2.InvalidRangeException; import ucar.nc2.NetcdfFile; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintWriter; +import java.io.*; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; @@ -31,9 +32,6 @@ class MmdQCTool { private final static Logger logger = FiduceoLogger.getLogger(); - private FileMessages fileMessages; - private MatchupAccumulator matchupAccumulator; - // package access for testing only tb 2023-02-14 static Options getOptions() { final Options options = new Options(); @@ -45,10 +43,22 @@ static Options getOptions() { inputOption.setRequired(true); options.addOption(inputOption); + final Option outputOption = new Option("o", "outdir", true, "Defines the result output directory."); + options.addOption(outputOption); + final Option timeOption = new Option("t", "time", true, "Defines matchup time variable name."); timeOption.setRequired(true); options.addOption(timeOption); + final Option plotOption = new Option("p", "plot", false, "Enables plotting the matchup locations onto a global map. Requires 'lon' and 'lat' to be set."); + options.addOption(plotOption); + + final Option lonOption = new Option("lon", "longitude", true, "Defines the variable name for the longitude."); + options.addOption(lonOption); + + final Option latOption = new Option("lat", "latitude", true, "Defines the variable name for the latitude."); + options.addOption(latOption); + return options; } @@ -68,7 +78,14 @@ static void writeReport(OutputStream outputStream, MatchupAccumulator accumulato final int size = messageMap.size(); writer.println(size + " file(s) with errors"); if (size > 0) { - // @todo 1 tb/tb add error messages per file here + for (Map.Entry> entry : messageMap.entrySet()) { + writer.println(entry.getKey()); + + final List messages = entry.getValue(); + for (final String message : messages) { + writer.println(" - " + message); + } + } } writer.println(); @@ -89,13 +106,21 @@ void run(CommandLine commandLine) throws IOException { final List mmdFiles = getInputFiles(inputDirOption); logger.info("Found " + mmdFiles.size() + " input file(s) to analyze."); - fileMessages = new FileMessages(); - matchupAccumulator = new MatchupAccumulator(); + final String outDirString = commandLine.getOptionValue("o"); + File outDir; + if (StringUtils.isNullOrEmpty(outDirString)) { + outDir = new File("."); + } else { + outDir = new File(outDirString); + } + + final FileMessages fileMessages = new FileMessages(); + final MatchupAccumulator matchupAccumulator = new MatchupAccumulator(); // loop over files for (final Path mmdFile : mmdFiles) { - try (final NetcdfFile netcdfFile = NetCDFUtils.openReadOnly(mmdFile.toAbsolutePath().toString())) { + matchupAccumulator.countFile(); // read time variable center pixel final String timeVariableName = commandLine.getOptionValue("t"); @@ -106,17 +131,62 @@ void run(CommandLine commandLine) throws IOException { final int time = iterator.getIntNext(); matchupAccumulator.add(time); } + + if (commandLine.hasOption("p")) { + plotMatchups(mmdFile, netcdfFile, commandLine, outDir); + } + } catch (IOException | InvalidRangeException ioException) { fileMessages.add(mmdFile.getFileName().toString(), ioException.getMessage()); } } - // write report - final TreeMap treeMap = new TreeMap<>(matchupAccumulator.getDaysMap()); - final Set> entries = treeMap.entrySet(); - for (final Map.Entry entry : entries) { - System.out.println(entry.getKey() + ": " + entry.getValue()); + final Date now = TimeUtils.createNow(); + final String timeString = TimeUtils.format(now, "yyyy-MM-dd"); + final String qcFileName = "mmd_qc_report_" + timeString + ".txt"; + final File reportFile = new File(outDir, qcFileName); + if (!reportFile.createNewFile()) { + throw new IOException("unable to create report file: " + reportFile.getAbsolutePath()); + } + try (FileOutputStream outStream = new FileOutputStream(reportFile)) { + writeReport(outStream, matchupAccumulator, fileMessages); + } + } + + private static void plotMatchups(Path mmdFile, NetcdfFile netcdfFile, CommandLine commandLine, File outDir) throws IOException, InvalidRangeException { + final String lonVariable = commandLine.getOptionValue("lon"); + final String latVariable = commandLine.getOptionValue("lat"); + + if (StringUtils.isNullOrEmpty(lonVariable) || StringUtils.isNullOrEmpty(latVariable)) { + throw new IllegalArgumentException("must provide lon and lat variable names for plotting"); + } + + final String filenameWithoutExtension = FileUtils.getFilenameWithoutExtension(mmdFile.toFile()); + final String pngFile = filenameWithoutExtension.concat(".png"); + final File pngFilePath = new File(outDir, pngFile); + + final GlobalPlot filePlot = GlobalPlot.create(); + final Array latitudes = NetCDFUtils.getCenterPosArrayFromMMDFile(netcdfFile, latVariable, null, + null, FiduceoConstants.MATCHUP_COUNT); + + final Array longitudes = NetCDFUtils.getCenterPosArrayFromMMDFile(netcdfFile, lonVariable, null, + null, FiduceoConstants.MATCHUP_COUNT); + + int numMatches = 1; + int[] shape = latitudes.getShape(); + if (shape.length > 0) { + numMatches = shape[0]; } + + final ArrayList pointList = new ArrayList<>(); + for (int i = 0; i < numMatches; i++) { + final GeoPos geoPos = new GeoPos(latitudes.getFloat(i), longitudes.getFloat(i)); + pointList.add(geoPos); + } + + filePlot.plot(pointList); + filePlot.writeTo(pngFilePath.getAbsolutePath()); + filePlot.dispose(); } private List getInputFiles(String inputDirOption) throws IOException { @@ -140,7 +210,7 @@ static void printUsageTo(OutputStream outputStream) { writer.write(ls + ls); final HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.printHelp(writer, 120, "matchup-tool ", "Valid options are:", + helpFormatter.printHelp(writer, 120, "mmd-qc-tool ", "Valid options are:", getOptions(), 3, 3, ""); writer.flush(); diff --git a/core/src/main/java/com/bc/fiduceo/qc/MmdQCToolMain.java b/core/src/main/java/com/bc/fiduceo/qc/MmdQCToolMain.java index 7415a53c7..f012cb416 100644 --- a/core/src/main/java/com/bc/fiduceo/qc/MmdQCToolMain.java +++ b/core/src/main/java/com/bc/fiduceo/qc/MmdQCToolMain.java @@ -1,10 +1,7 @@ package com.bc.fiduceo.qc; import com.bc.fiduceo.log.FiduceoLogger; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; +import org.apache.commons.cli.*; public class MmdQCToolMain { @@ -16,7 +13,7 @@ public static void main(String[] args) throws ParseException { return; } - final CommandLineParser parser = new PosixParser(); + final CommandLineParser parser = new DefaultParser(); final CommandLine commandLine = parser.parse(MmdQCTool.getOptions(), args); if (commandLine.hasOption("h") || commandLine.hasOption("--help")) { MmdQCTool.printUsageTo(System.err); diff --git a/core/src/main/java/com/bc/fiduceo/reader/RawDataReader.java b/core/src/main/java/com/bc/fiduceo/reader/RawDataReader.java index 13d8f2a5d..e2f8a3aa8 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/RawDataReader.java +++ b/core/src/main/java/com/bc/fiduceo/reader/RawDataReader.java @@ -25,6 +25,7 @@ import ucar.ma2.*; import java.awt.*; +import java.awt.geom.Rectangle2D; import java.io.IOException; /** @@ -131,12 +132,18 @@ private static Array readFrom1DArray(int offsetX, int offsetY, int windowWidth, } } - private static boolean isWindowInside(int winOffSetX, int winOffSetY, int windowWidth, int windowHeight, int rawWidth, int rawHeight) { + public static boolean isWindowInside(int winOffSetX, int winOffSetY, int windowWidth, int windowHeight, int rawWidth, int rawHeight) { final Rectangle windowRec = new Rectangle(winOffSetX, winOffSetY, windowWidth, windowHeight); final Rectangle arrayRectangle = new Rectangle(0, 0, rawWidth, rawHeight); return arrayRectangle.contains(windowRec); } + public static Rectangle2D getInsideWindow(int winOffSetX, int winOffSetY, int windowWidth, int windowHeight, int rawWidth, int rawHeight) { + final Rectangle windowRec = new Rectangle(winOffSetX, winOffSetY, windowWidth, windowHeight); + final Rectangle arrayRectangle = new Rectangle(0, 0, rawWidth, rawHeight); + return arrayRectangle.createIntersection(windowRec); + } + // package access for testing only tb 2016-04-18 static InputDimension getInputDimension(int rank, int[] shape) { if (rank == 1 && shape[0] == 1) { diff --git a/core/src/main/java/com/bc/fiduceo/reader/ReaderPlugin.java b/core/src/main/java/com/bc/fiduceo/reader/ReaderPlugin.java index 33dba6e39..6e67dc426 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/ReaderPlugin.java +++ b/core/src/main/java/com/bc/fiduceo/reader/ReaderPlugin.java @@ -1,7 +1,5 @@ package com.bc.fiduceo.reader; -import com.bc.fiduceo.geometry.GeometryFactory; - public interface ReaderPlugin { /** diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_Reader.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_Reader.java new file mode 100644 index 000000000..ccdc86269 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_Reader.java @@ -0,0 +1,54 @@ +package com.bc.fiduceo.reader.amsu_mhs; + +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.reader.*; +import com.bc.fiduceo.reader.amsu_mhs.nat.*; +import com.bc.fiduceo.reader.amsu_mhs.nat.record_types.MDR; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +import static com.bc.fiduceo.reader.amsu_mhs.nat.EPS_Constants.*; + +public class AMSUA_L1B_Reader extends Abstract_L1B_NatReader { + + public static final String RESOURCE_KEY = "AMSUA_L1B"; + + AMSUA_L1B_Reader(ReaderContext readerContext) { + super(readerContext); + } + + @Override + public void open(File file) throws IOException { + initializeRegistry(RESOURCE_KEY); + readDataToCache(file, EPS_Constants.AMSUA_FOV_COUNT); + + final List mdrs = cache.getMdrs(); + ensureMdrVersionSupported(mdrs.get(0).getHeader()); + } + + @Override + public AcquisitionInfo read() throws IOException { + return super.read(new Interval(6, 20)); + } + + @Override + public String getRegEx() { + return "AMSA_[A-Z0-9x]{3}_1B_M0[123]_[0-9]{14}Z_[0-9]{14}Z_[A-Z0-9x]{1}_[A-Z0-9x]{1}_[0-9]{14}Z\\.nat"; + } + + @Override + public Dimension getProductSize() throws IOException { + return super.getProductSize(AMSUA_FOV_COUNT); + } + + static void ensureMdrVersionSupported(GENERIC_RECORD_HEADER header) { + final byte recordSubClass = header.getRecordSubClass(); + final byte recordSubClassVersion = header.getRecordSubClassVersion(); + if (recordSubClass != 2 || recordSubClassVersion != 3) { + throw new IllegalStateException("Unsupported MDR version: " + recordSubClass + " v " + recordSubClassVersion); + } + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_ReaderPlugin.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_ReaderPlugin.java new file mode 100644 index 000000000..7283e9085 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_ReaderPlugin.java @@ -0,0 +1,26 @@ +package com.bc.fiduceo.reader.amsu_mhs; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.ReaderContext; +import com.bc.fiduceo.reader.ReaderPlugin; + +public class AMSUA_L1B_ReaderPlugin implements ReaderPlugin { + + private static final String[] SENSOR_KEYS = {"amsua-ma-l1b", "amsua-mb-l1b", "amsua-mc-l1b"}; + + @Override + public Reader createReader(ReaderContext readerContext) { + return new AMSUA_L1B_Reader(readerContext); + } + + @Override + public String[] getSupportedSensorKeys() { + return SENSOR_KEYS; + } + + @Override + public DataType getDataType() { + return DataType.POLAR_ORBITING_SATELLITE; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_Reader.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_Reader.java new file mode 100644 index 000000000..ed4abf758 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_Reader.java @@ -0,0 +1,41 @@ +package com.bc.fiduceo.reader.amsu_mhs; + +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.reader.*; +import com.bc.fiduceo.reader.amsu_mhs.nat.*; + +import java.io.File; +import java.io.IOException; + +import static com.bc.fiduceo.reader.amsu_mhs.nat.EPS_Constants.*; + +public class MHS_L1B_Reader extends Abstract_L1B_NatReader { + + public static final String RESOURCE_KEY = "MHS_L1B"; + + MHS_L1B_Reader(ReaderContext readerContext) { + super(readerContext); + } + + @Override + public void open(File file) throws IOException { + initializeRegistry(RESOURCE_KEY); + readDataToCache(file, EPS_Constants.MHS_FOV_COUNT); + } + + @Override + public AcquisitionInfo read() throws IOException { + return super.read(new Interval(10, 20)); + } + + @Override + public String getRegEx() { + return "MHSx_[A-Z0-9x]{3}_1B_M0[123]_[0-9]{14}Z_[0-9]{14}Z_[A-Z0-9x]{1}_[A-Z0-9x]{1}_[0-9]{14}Z\\.nat"; + } + + @Override + public Dimension getProductSize() throws IOException { + return super.getProductSize(MHS_FOV_COUNT); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_ReaderPlugin.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_ReaderPlugin.java new file mode 100644 index 000000000..6ed23b0c8 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_ReaderPlugin.java @@ -0,0 +1,26 @@ +package com.bc.fiduceo.reader.amsu_mhs; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.ReaderContext; +import com.bc.fiduceo.reader.ReaderPlugin; + +public class MHS_L1B_ReaderPlugin implements ReaderPlugin { + + private static final String[] SENSOR_KEYS = {"mhs-ma-l1b", "mhs-mb-l1b", "mhs-mc-l1b"}; + + @Override + public Reader createReader(ReaderContext readerContext) { + return new MHS_L1B_Reader(readerContext); + } + + @Override + public String[] getSupportedSensorKeys() { + return SENSOR_KEYS; + } + + @Override + public DataType getDataType() { + return DataType.POLAR_ORBITING_SATELLITE; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/Abstract_L1B_NatReader.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/Abstract_L1B_NatReader.java new file mode 100644 index 000000000..99f636ec8 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/Abstract_L1B_NatReader.java @@ -0,0 +1,313 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.geometry.Geometry; +import com.bc.fiduceo.geometry.GeometryFactory; +import com.bc.fiduceo.geometry.Polygon; +import com.bc.fiduceo.location.PixelGeoCodingPixelLocator; +import com.bc.fiduceo.location.PixelLocator; +import com.bc.fiduceo.reader.*; +import com.bc.fiduceo.reader.amsu_mhs.nat.record_types.MPHR; +import com.bc.fiduceo.reader.time.TimeLocator; +import com.bc.fiduceo.reader.time.TimeLocator_StartStopDate; +import com.bc.fiduceo.util.NetCDFUtils; +import com.bc.fiduceo.util.VariableProxy; +import org.esa.snap.core.dataio.geocoding.GeoChecks; +import org.esa.snap.core.util.StringUtils; +import ucar.ma2.Array; +import ucar.ma2.ArrayInt; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.*; + +import static com.bc.fiduceo.core.NodeType.UNDEFINED; +import static com.bc.fiduceo.reader.amsu_mhs.nat.EPS_Constants.*; +import static ucar.ma2.DataType.INT; + +abstract public class Abstract_L1B_NatReader implements Reader { + + protected VariableRegistry registry; + protected EpsVariableCache cache; + protected final GeometryFactory geometryFactory; + private Dimension productSize; + private static final int NUM_SPLITS = 2; + + private PixelLocator pixelLocator; + + public Abstract_L1B_NatReader(ReaderContext readerContext) { + this.geometryFactory = readerContext.getGeometryFactory(); + pixelLocator = null; + productSize = null; + } + + public AcquisitionInfo read(Interval interval) throws IOException { + final AcquisitionInfo acquisitionInfo = new AcquisitionInfo(); + acquisitionInfo.setNodeType(UNDEFINED); + + final MPHR recordMPHR = cache.getMPHR(); + setSensingDates(acquisitionInfo, recordMPHR); + + final Array lon = cache.getScaled(LON_VAR_NAME); + final Array lat = cache.getScaled(LAT_VAR_NAME); + + final Geometries geometries = extractGeometries(lon, lat, NUM_SPLITS, interval); + acquisitionInfo.setBoundingGeometry(geometries.getBoundingGeometry()); + ReaderUtils.setTimeAxes(acquisitionInfo, geometries.getTimeAxesGeometry(), geometryFactory); + + return acquisitionInfo; + } + + @Override + public Array readRaw(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException { + final Array rawData = cache.getRaw(variableName); + final VariableDefinition variableDef = registry.getVariableDef(variableName); + final Number fillValue = EpsReaderUtils.getFillValue(variableDef.getData_type()); + final Dimension productSize = getProductSize(); + return RawDataReader.read(centerX, centerY, interval, fillValue, rawData, productSize); + } + + @Override + public Array readScaled(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException { + final Array rawData = readRaw(centerX, centerY, interval, variableName); + final VariableDefinition variableDef = registry.getVariableDef(variableName); + return EpsReaderUtils.scale(rawData, variableDef.getScale_factor()); + } + + @Override + public String getLongitudeVariableName() { + return LON_VAR_NAME; + } + + @Override + public String getLatitudeVariableName() { + return LAT_VAR_NAME; + } + + @Override + public PixelLocator getPixelLocator() throws IOException { + if (pixelLocator == null) { + final Array lon = cache.getScaled(LON_VAR_NAME); + final Array lat = cache.getScaled(LAT_VAR_NAME); + + final PixelGeoCodingPixelLocator pixelGeoCodingPixelLocator = new PixelGeoCodingPixelLocator(lon, lat, LON_VAR_NAME, LAT_VAR_NAME, 48.0, GeoChecks.POLES); + pixelGeoCodingPixelLocator.setHasHalfPxOffset(false); + pixelLocator = pixelGeoCodingPixelLocator; + } + return pixelLocator; + } + + @Override + public PixelLocator getSubScenePixelLocator(Polygon sceneGeometry) throws IOException { + return getPixelLocator(); + } + + @Override + public TimeLocator getTimeLocator() throws IOException { + // for the test file, the array returned contains only zeros - same as for MHS + // According to the sparse documentation, I would expect this to contain seconds since epoch + // tb 2025-09-04 + // Array timeAttitude = cache.getRaw("TIME_ATTITUDE"); + + // Instead, we interpolate between header start and stop times tb 2025-09-04 + final MPHR mphr = cache.getMPHR(); + final Date sensingStart = mphr.getSensingStart(); + final Date sensingStop = mphr.getSensingStop(); + + return new TimeLocator_StartStopDate(sensingStart, sensingStop, getProductSize().getNy()); + } + + @Override + public int[] extractYearMonthDayFromFilename(String fileName) { + final String[] strings = fileName.split("_"); + final String dateTimePart = strings[4]; + + final int[] ymd = new int[3]; + ymd[0] = Integer.parseInt(dateTimePart.substring(0, 4)); + ymd[1] = Integer.parseInt(dateTimePart.substring(4, 6)); + ymd[2] = Integer.parseInt(dateTimePart.substring(6, 8)); + return ymd; + } + + protected static void setSensingDates(AcquisitionInfo acquisitionInfo, MPHR recordMPHR) throws IOException { + final Date sensingStart = recordMPHR.getSensingStart(); + acquisitionInfo.setSensingStart(sensingStart); + final Date sensingEnd = recordMPHR.getSensingStop(); + acquisitionInfo.setSensingStop(sensingEnd); + } + + protected void initializeRegistry(String resourceKey) { + registry = VariableRegistry.load(resourceKey); + } + + protected void readDataToCache(File file, int sensorKey) throws IOException { + final byte[] rawDataBuffer; + try (FileInputStream fis = new FileInputStream(file)) { + rawDataBuffer = fis.readAllBytes(); + } + cache = new EpsVariableCache(rawDataBuffer, registry, sensorKey); + } + + @Override + public void close() throws IOException { + if (cache != null) { + cache.clear(); + cache = null; + } + if (registry != null) { + registry.clear(); + registry = null; + } + pixelLocator = null; + productSize = null; + } + + @Override + public List getVariables() throws InvalidRangeException, IOException { + final ArrayList variables = new ArrayList<>(); + + final Map regVariables = registry.getVariables(); + final Set keySet = regVariables.keySet(); + for (String variableName : keySet) { + final VariableDefinition variableDefinition = regVariables.get(variableName); + final int productDataType = variableDefinition.getProductData_type(); + final DataType netcdfDataType = NetCDFUtils.getNetcdfDataType(productDataType); + final List attributes = extractCFAttributes(variableDefinition); + + final VariableProxy variable = new VariableProxy(variableName, netcdfDataType, attributes); + variables.add(variable); + } + + return variables; + } + + @Override + public ArrayInt.D2 readAcquisitionTime(int x, int y, Interval interval) throws IOException, InvalidRangeException { + final int width = interval.getX(); + final int height = interval.getY(); + final int[] timeArray = new int[width * height]; + + final Dimension size = getProductSize(); + final int sceneRasterHeight = size.getNy(); + final int sceneRasterWidth = size.getNx(); + final int halfHeight = height / 2; + final int halfWidth = width / 2; + int writeOffset = 0; + final int fillValue = NetCDFUtils.getDefaultFillValue(int.class).intValue(); + final TimeLocator timeLocator = getTimeLocator(); + + for (int yRead = y - halfHeight; yRead <= y + halfHeight; yRead++) { + int lineTimeSeconds = fillValue; + if (yRead >= 0 && yRead < sceneRasterHeight) { + final long lineTimeMillis = timeLocator.getTimeFor(x, yRead); + lineTimeSeconds = (int) Math.round(lineTimeMillis * 0.001); + } + + for (int xRead = x - halfWidth; xRead <= x + halfWidth; xRead++) { + if (xRead >= 0 && xRead < sceneRasterWidth) { + timeArray[writeOffset] = lineTimeSeconds; + } else { + timeArray[writeOffset] = fillValue; + } + ++writeOffset; + } + } + + final int[] shape = new int[]{interval.getY(), interval.getX()}; + return (ArrayInt.D2) Array.factory(INT, shape, timeArray); + } + + public Dimension getProductSize(int fov_count) { + if (productSize == null) { + final int numScanLines = cache.getMdrs().size(); + productSize = new Dimension("size", fov_count, numScanLines); + } + return productSize; + } + + // public access for testing only tb 2025-09-17 + public static List extractCFAttributes(VariableDefinition variableDefinition) { + final ArrayList attributes = new ArrayList<>(); + + final String units = variableDefinition.getUnits(); + if (StringUtils.isNotNullAndNotEmpty(units)) { + attributes.add(new Attribute("units", units)); + } + + final double scaleFactor = variableDefinition.getScale_factor(); + if (scaleFactor != 1.0) { + attributes.add(new Attribute("scale_factor", scaleFactor)); + attributes.add(new Attribute("add_offset", 0.0)); + } + + final String dataType = variableDefinition.getData_type(); + if (StringUtils.isNotNullAndNotEmpty(dataType)) { + final Number fillValue = EpsReaderUtils.getFillValue(dataType); + if (fillValue != null) { + attributes.add(new Attribute("_FillValue", fillValue)); + } + } + + final String flagMeanings = variableDefinition.getFlag_meanings(); + final String flagValues = variableDefinition.getFlag_values(); + if (StringUtils.isNotNullAndNotEmpty(flagMeanings) && StringUtils.isNotNullAndNotEmpty(flagValues)) { + attributes.add(new Attribute("flag_meanings", flagMeanings)); + + final Array valuesArray = toValuesArray(flagValues, variableDefinition.getData_type()); + attributes.add(new Attribute("flag_values", valuesArray)); + } + + final String standardName = variableDefinition.getStandard_name(); + if (StringUtils.isNotNullAndNotEmpty(standardName)) { + attributes.add(new Attribute("standard_name", standardName)); + } + + return attributes; + } + + // public access for testing only tb 2025-09-17 + public static Array toValuesArray(String valuesString, String dataType) { + final String[] valueStrings = StringUtils.split(valuesString, new char[]{','}, true); + final int snapDataType = EpsReaderUtils.mapToProductData(dataType); + + Array dataVector = Array.factory(NetCDFUtils.getNetcdfDataType(snapDataType), new int[]{valueStrings.length}); + + for (int i = 0; i < valueStrings.length; i++) { + dataVector.setDouble(i, Double.parseDouble(valueStrings[i])); + } + return dataVector; + } + + protected Geometries extractGeometries(Array longitudes, Array latitudes, int numSplits, Interval interval) throws IOException { + final Geometries geometries = new Geometries(); + final BoundingPolygonCreator boundingPolygonCreator = getBoundingPolygonCreator(interval); + + Geometry boundingGeometry = boundingPolygonCreator.createBoundingGeometryClockwise(longitudes, latitudes); + Geometry timeAxisGeometry; + + if (!boundingGeometry.isValid()) { + boundingGeometry = boundingPolygonCreator.createBoundingGeometrySplitted(longitudes, latitudes, numSplits, true); + if (!boundingGeometry.isValid()) { + throw new RuntimeException("Invalid bounding geometry detected"); + } + timeAxisGeometry = boundingPolygonCreator.createTimeAxisGeometrySplitted(longitudes, latitudes, numSplits); + } else { + timeAxisGeometry = boundingPolygonCreator.createTimeAxisGeometry(longitudes, latitudes); + } + + geometries.setBoundingGeometry(boundingGeometry); + geometries.setTimeAxesGeometry(timeAxisGeometry); + + return geometries; + } + + private BoundingPolygonCreator getBoundingPolygonCreator(Interval interval) { + return new BoundingPolygonCreator(interval, geometryFactory); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/DATA_LAYOUT.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/DATA_LAYOUT.java new file mode 100644 index 000000000..c628aa044 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/DATA_LAYOUT.java @@ -0,0 +1,27 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +public enum DATA_LAYOUT { + + VECTOR, + ARRAY; + + public static DATA_LAYOUT fromString(String dataLayout) { + if (dataLayout.equals("ARRAY")) { + return DATA_LAYOUT.ARRAY; + } else if (dataLayout.equals("VECTOR")) { + return DATA_LAYOUT.VECTOR; + } else { + throw new IllegalArgumentException("Unsupported data layout: " + dataLayout); + } + } + + public static String toString(DATA_LAYOUT dataLayout) { + if (dataLayout == DATA_LAYOUT.VECTOR) { + return "VECTOR"; + } else if (dataLayout == DATA_LAYOUT.ARRAY) { + return "ARRAY"; + } else { + throw new IllegalArgumentException("Unsupported data layout: " + dataLayout); + } + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/EPS_Constants.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/EPS_Constants.java new file mode 100644 index 000000000..a51519b4c --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/EPS_Constants.java @@ -0,0 +1,19 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +public class EPS_Constants { + + public static final int GENERIC_RECORD_HEADER_SIZE = 20; + + public static final int MHS_FOV_COUNT = 90; + public static final int MHS_L1B_EARTH_LOCATIONS_OFFSET = 3318; + public static final int MHS_EARTH_LOCATIONS_TOTAL_BYTE_SIZE = 720; + public static final int MHS_EARTH_LOCATIONS_SCALE_FACTOR = 10000; + + public static final int AMSUA_FOV_COUNT = 30; + + public static String SENSING_START_KEY = "SENSING_START"; + public static String SENSING_STOP_KEY = "SENSING_END"; + + public static String LON_VAR_NAME = "longitude"; + public static String LAT_VAR_NAME = "latitude"; +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsReaderUtils.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsReaderUtils.java new file mode 100644 index 000000000..08836ce0b --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsReaderUtils.java @@ -0,0 +1,164 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import com.bc.fiduceo.reader.ReaderUtils; +import org.esa.snap.core.datamodel.ProductData; +import ucar.ma2.*; + +import java.math.BigInteger; +import java.nio.ByteBuffer; + +public class EpsReaderUtils { + + /** + * reads eps data types "byte", "boolean", "enumerated" from buffer + */ + public static byte readInt8(ByteBuffer buffer, int offset) { + return buffer.get(offset); + } + + /** + * reads eps data types "u-byte" from buffer + */ + public static short readUInt8(ByteBuffer buffer, int offset) { + return (short) Byte.toUnsignedInt(buffer.get(offset)); + } + + /** + * reads eps data types "integer2" from buffer + */ + public static short readInt16(ByteBuffer buffer, int offset) { + return buffer.getShort(offset); + } + + /** + * reads eps data types "u-integer2" from buffer + */ + public static int readUInt16(ByteBuffer buffer, int offset) { + return Short.toUnsignedInt(buffer.getShort(offset)); + } + + /** + * reads eps data types "integer-4" from buffer + */ + public static int readInt32(ByteBuffer buffer, int offset) { + return buffer.getInt(offset); + } + + /** + * reads eps data types "u-integer4" from buffer + */ + public static long readUInt32(ByteBuffer buffer, int offset) { + return Integer.toUnsignedLong(buffer.getInt(offset)); + } + + /** + * reads eps data types "integer8" from buffer + */ + public static long readInt64(ByteBuffer buffer, int offset) { + return buffer.getLong(offset); + } + + /** + * reads eps data types "u-integer8" from buffer + */ + public static BigInteger readUInt64(ByteBuffer buffer, int offset) { + byte[] bytes = new byte[8]; + buffer.position(offset); + buffer.get(bytes); + return new BigInteger(1, bytes); // unsigned interpretation + } + + public static int mapToProductData(String value) { + switch (value.toLowerCase()) { + case "byte": + case "boolean": + case "enumerated": + return ProductData.TYPE_INT8; + case "u-byte": + return ProductData.TYPE_UINT8; + case "integer2": + return ProductData.TYPE_INT16; + case "u-integer2": + return ProductData.TYPE_UINT16; + case "integer4": + return ProductData.TYPE_INT32; + case "u-integer4": + return ProductData.TYPE_UINT32; + case "integer8": + return ProductData.TYPE_INT64; + case "u-integer8": + return ProductData.TYPE_UINT64; + default: + throw new IllegalArgumentException("Unknown data type: " + value); + } + } + + public static Array scale(Array array, double scaleFactor) { + if (ReaderUtils.mustScale(scaleFactor, 0.0)) { + final MAMath.ScaleOffset scaleOffset = new MAMath.ScaleOffset(scaleFactor, 0.0); + return MAMath.convert2Unpacked(array, scaleOffset); + } + return array; + } + + public static Array initializeArray(int dataType, int numScanLines, int numFOVs) { + Array array; + + switch (dataType) { + case ProductData.TYPE_INT8: + array = new ArrayByte.D2(numScanLines, numFOVs, false); + break; + case ProductData.TYPE_UINT8: + array = new ArrayByte.D2(numScanLines, numFOVs, true); + break; + case ProductData.TYPE_INT16: + array = new ArrayShort.D2(numScanLines, numFOVs, false); + break; + case ProductData.TYPE_UINT16: + array = new ArrayShort.D2(numScanLines, numFOVs, true); + break; + case ProductData.TYPE_INT32: + array = new ArrayInt.D2(numScanLines, numFOVs, false); + break; + case ProductData.TYPE_UINT32: + array = new ArrayInt.D2(numScanLines, numFOVs, true); + break; + case ProductData.TYPE_INT64: + array = new ArrayLong.D2(numScanLines, numFOVs, false); + break; + case ProductData.TYPE_UINT64: + array = new ArrayLong.D2(numScanLines, numFOVs, true); + break; + default: + array = new ArrayDouble.D2(numScanLines, numFOVs); + } + + if (numFOVs == 1) { + array = array.reduce(); + } + return array; + } + + public static Number getFillValue(String dataType) { + switch (dataType) { + case "byte": + return Byte.MIN_VALUE; + case "u-byte": + return 255; + case "integer2": + return Short.MIN_VALUE; + case "u-integer2": + return 65535; + case "integer4": + return Integer.MIN_VALUE; + case "u-integer4": + return 4294967295L; + case "integer8": + return Long.MIN_VALUE; + case "u-integer8": + return Long.MAX_VALUE; // @todo 2 tb/* check how we can handle uint64 in Java anyways 2025-09-16 + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsVariableCache.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsVariableCache.java new file mode 100644 index 000000000..df4809f8f --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsVariableCache.java @@ -0,0 +1,128 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import com.bc.fiduceo.reader.ReaderUtils; +import com.bc.fiduceo.reader.amsu_mhs.nat.record_types.MDR; +import com.bc.fiduceo.reader.amsu_mhs.nat.record_types.MPHR; +import org.esa.snap.core.datamodel.ProductData; +import ucar.ma2.*; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class EpsVariableCache { + + private final int numFOVs; + private final List records; + private final VariableRegistry registry; + + private final Map rawDataCache; + + public EpsVariableCache(byte[] rawDataBuffer, VariableRegistry registry, int numFOVs) { + this.records = RecordFactory.parseRecords(rawDataBuffer); + this.registry = registry; + this.rawDataCache = new HashMap<>(); + this.numFOVs = numFOVs; + } + + public void clear() { + rawDataCache.clear(); + } + + public MPHR getMPHR() { + return MdrUtilities.getMphr(records); + } + + public List getMdrs() { + // @todo 2 tb this should be a lazy-loaded instance + return MdrUtilities.getMdrList(records); + } + + public Array getRaw(String variableName) { + Array array = rawDataCache.get(variableName); + + if (array == null) { + array = decodeRawDataArray(variableName); + rawDataCache.put(variableName, array); + } + + return array; + } + + public Array getScaled(String variableName) { + final Array rawArray = getRaw(variableName); + + final VariableDefinition varDef = registry.getVariableDef(variableName); + double scaleFactor = varDef.getScale_factor(); + return EpsReaderUtils.scale(rawArray, scaleFactor); + } + + private Array decodeRawDataArray(String variableName) { + Array array; + VariableDefinition varDef = registry.getVariableDef(variableName); + List mdrs = getMdrs(); + int numScanLines = mdrs.size(); + int stride = varDef.getStride(); + int offset = varDef.getOffset(); + int dataType = varDef.getProductData_type(); + int size = ProductData.getElemSize(dataType); + + DATA_LAYOUT dataLayout = DATA_LAYOUT.fromString(varDef.getData_layout()); + if (dataLayout == DATA_LAYOUT.ARRAY) { + array = EpsReaderUtils.initializeArray(dataType, numScanLines, numFOVs); + + for (int yy = 0; yy < numScanLines; yy++) { + MDR mdr = mdrs.get(yy); + byte[] payload = mdr.getPayload(); + + for (int xx = 0; xx < numFOVs; xx++) { + int valueSpecificOffset = offset + xx * stride * size; + // @todo 1 tb/tb why as double??? + double value = readValue(payload, valueSpecificOffset, varDef); + + array.setDouble(array.getIndex().set(yy, xx), value); + } + } + return array; + } else if (dataLayout == DATA_LAYOUT.VECTOR) { + array = EpsReaderUtils.initializeArray(dataType, numScanLines, 1); + for (int y = 0; y < numScanLines; y++) { + MDR mdr = mdrs.get(y); + byte[] payload = mdr.getPayload(); + double value = readValue(payload, offset, varDef); + array.setDouble(array.getIndex().set(y), value); + } + return array; + } else { + throw new IllegalStateException("Unsupported data layout: " + dataLayout); + } + } + + private double readValue(byte[] payload, int offset, VariableDefinition def) { + ByteBuffer buffer = ByteBuffer.wrap(payload).order(ByteOrder.BIG_ENDIAN); + int type = def.getProductData_type(); + + switch (type) { + case ProductData.TYPE_INT8: + return EpsReaderUtils.readInt8(buffer, offset); + case ProductData.TYPE_UINT8: + return EpsReaderUtils.readUInt8(buffer, offset); + case ProductData.TYPE_INT16: + return EpsReaderUtils.readInt16(buffer, offset); + case ProductData.TYPE_UINT16: + return EpsReaderUtils.readUInt16(buffer, offset); + case ProductData.TYPE_INT32: + return EpsReaderUtils.readInt32(buffer, offset); + case ProductData.TYPE_UINT32: + return EpsReaderUtils.readUInt32(buffer, offset); + case ProductData.TYPE_INT64: + return EpsReaderUtils.readInt64(buffer, offset); + case ProductData.TYPE_UINT64: + return EpsReaderUtils.readUInt64(buffer, offset).doubleValue(); + default: + throw new IllegalArgumentException("Unsupported data type: " + type); + } + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/GENERIC_RECORD_HEADER.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/GENERIC_RECORD_HEADER.java new file mode 100644 index 000000000..b798e301f --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/GENERIC_RECORD_HEADER.java @@ -0,0 +1,51 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import java.nio.ByteBuffer; + +public class GENERIC_RECORD_HEADER { + + private final RECORD_CLASS recordClass; + private final INSTRUMENT_GROUP instrumentGroup; + private final int recordSize; + private final byte recordSubClass; + private final byte recordSubClassVersion; + + GENERIC_RECORD_HEADER(RECORD_CLASS recordClass, INSTRUMENT_GROUP instrumentGroup, byte recordSubClass, byte recordSubClassVersion, int recordSize) { + this.recordClass = recordClass; + this.instrumentGroup = instrumentGroup; + this.recordSize = recordSize; + this.recordSubClass = recordSubClass; + this.recordSubClassVersion = recordSubClassVersion; + } + + public static GENERIC_RECORD_HEADER parse(byte[] data) { + final RECORD_CLASS recordClass = RECORD_CLASS.fromByte(data[0]); + final INSTRUMENT_GROUP instrumentGroup = INSTRUMENT_GROUP.fromByte(data[1]); + final byte recordSubClass = data[2]; + final byte recordSubClassVersion = data[3]; + + final ByteBuffer buffer = ByteBuffer.wrap(data,4,4); + final int recordSize = buffer.getInt(); + return new GENERIC_RECORD_HEADER(recordClass, instrumentGroup, recordSubClass,recordSubClassVersion, recordSize); + } + + public RECORD_CLASS getRecordClass() { + return recordClass; + } + + public INSTRUMENT_GROUP getInstrumentGroup() { + return instrumentGroup; + } + + public int getRecordSize() { + return recordSize; + } + + public byte getRecordSubClass() { + return recordSubClass; + } + + public byte getRecordSubClassVersion() { + return recordSubClassVersion; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/INSTRUMENT_GROUP.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/INSTRUMENT_GROUP.java new file mode 100644 index 000000000..6d14d05bb --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/INSTRUMENT_GROUP.java @@ -0,0 +1,59 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +public enum INSTRUMENT_GROUP { + GENERIC, + AMSUA, + ASCAT, + ATOVS, + AVHRR3, + GOME, + GRAS, + HIRS4, + IASI, + MHS, + SEM, + ADCS, + SBUV, + DUMMY, + IASI_L2, + ARCHIVE; + + public static INSTRUMENT_GROUP fromByte(byte b) { + switch (b) { + case 0: + return GENERIC; + case 1: + return AMSUA; + case 2: + return ASCAT; + case 3: + return ATOVS; + case 4: + return AVHRR3; + case 5: + return GOME; + case 6: + return GRAS; + case 7: + return HIRS4; + case 8: + return IASI; + case 9: + return MHS; + case 10: + return SEM; + case 11: + return ADCS; + case 12: + return SBUV; + case 13: + return DUMMY; + case 15: + return IASI_L2; + case 99: + return ARCHIVE; + default: + throw new IllegalArgumentException("Unknown instrument group: " + b); + } + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/MdrUtilities.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/MdrUtilities.java new file mode 100644 index 000000000..5302a54dd --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/MdrUtilities.java @@ -0,0 +1,28 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import com.bc.fiduceo.reader.amsu_mhs.nat.record_types.MDR; +import com.bc.fiduceo.reader.amsu_mhs.nat.record_types.MPHR; + +import java.util.ArrayList; +import java.util.List; + +public class MdrUtilities { + + public static MPHR getMphr(List records) { + return (MPHR) records.get(0); + } + + public static List getMdrList(List records) { + List mdrs = new ArrayList<>(); + + for (Record record : records) { + GENERIC_RECORD_HEADER header = record.getHeader(); + RECORD_CLASS recordClass = header.getRecordClass(); + + if (recordClass == RECORD_CLASS.MDR) { + mdrs.add((MDR) record); + } + } + return mdrs; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/RECORD_CLASS.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/RECORD_CLASS.java new file mode 100644 index 000000000..47bd982f3 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/RECORD_CLASS.java @@ -0,0 +1,38 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +public enum RECORD_CLASS { + RESERVED, + MPHR, // Main Product Header Record + SPHR, // Secondary Product Header Record + IPR, // Internal Pointer Record + GEADR, // Global External Auxiliary Data Record + GIADR, // Global Internal Auxiliary Data Record + VEADR, // Variable External Auxiliary Data Record + VIADR, // Variable Internal Auxiliary Data Record + MDR; // Measurement Data Record + + public static RECORD_CLASS fromByte(byte b) { + switch (b) { + case 0: + return RESERVED; + case 1: + return MPHR; + case 2: + return SPHR; + case 3: + return IPR; + case 4: + return GEADR; + case 5: + return GIADR; + case 6: + return VEADR; + case 7: + return VIADR; + case 8: + return MDR; + default: + throw new IllegalArgumentException("Unknown record class: " + b); + } + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/Record.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/Record.java new file mode 100644 index 000000000..49dccebe8 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/Record.java @@ -0,0 +1,20 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +public class Record { + + private final GENERIC_RECORD_HEADER header; + private final byte[] payload; + + public Record(GENERIC_RECORD_HEADER header, byte[] payload) { + this.header = header; + this.payload = payload; + } + + public GENERIC_RECORD_HEADER getHeader() { + return header; + } + + public byte[] getPayload() { + return payload; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/RecordFactory.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/RecordFactory.java new file mode 100644 index 000000000..d1f74c43c --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/RecordFactory.java @@ -0,0 +1,64 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import com.bc.fiduceo.reader.amsu_mhs.nat.record_types.*; + +import java.util.ArrayList; +import java.util.List; + +import static com.bc.fiduceo.reader.amsu_mhs.nat.INSTRUMENT_GROUP.AMSUA; + +public class RecordFactory { + + public static List parseRecords(byte[] allBytes) { + List records = new ArrayList<>(); + int index = 0; + + while (index < allBytes.length) { + byte[] headerBytes = new byte[EPS_Constants.GENERIC_RECORD_HEADER_SIZE]; + System.arraycopy(allBytes, index, headerBytes, 0, EPS_Constants.GENERIC_RECORD_HEADER_SIZE); + + GENERIC_RECORD_HEADER header = GENERIC_RECORD_HEADER.parse(headerBytes); + int recordSize = header.getRecordSize(); + byte[] payload = new byte[recordSize]; + System.arraycopy(allBytes, index, payload, 0, recordSize); + + Record record = createRecord(header, payload); + records.add(record); + + index += recordSize; + } + + return records; + } + + public static Record createRecord(GENERIC_RECORD_HEADER header, byte[] payload) { + switch (header.getRecordClass()) { + case MPHR: + return new MPHR(header, payload); + case SPHR: + return new SPHR(header, payload); + case IPR: + return new IPR(header, payload); + case GEADR: + return new GEADR(header, payload); + case GIADR: + return new GIADR(header, payload); + case VEADR: + return new VEADR(header, payload); + case VIADR: + return new VIADR(header, payload); + case MDR: + return createMDR(header, payload); + default: + return new Record(header, payload); + } + } + + private static MDR createMDR(GENERIC_RECORD_HEADER header, byte[] payload) { + if (header.getInstrumentGroup() == AMSUA) { + return new ASMUSA_MDR(header, payload); + } + + return new MDR(header, payload); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableDefinition.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableDefinition.java new file mode 100644 index 000000000..6dcef4c06 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableDefinition.java @@ -0,0 +1,98 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +public class VariableDefinition { + + private String data_type; + private String data_layout; + private int offset; + private int stride; + private double scale_factor; + private String units; + private String flag_values; + private String flag_meanings; + private String standard_name; + + public VariableDefinition() { + scale_factor = 1.0; + units = ""; + data_layout = "ARRAY"; + offset = 0; + stride = 0; + } + + public int getProductData_type() { + return EpsReaderUtils.mapToProductData(data_type); + } + + public String getData_type() { + return data_type; + } + + public void setData_type(String data_type) { + this.data_type = data_type; + } + + public int getOffset() { + return offset; + } + + public void setOffset(int offset) { + this.offset = offset; + } + + public int getStride() { + return stride; + } + + public void setStride(int stride) { + this.stride = stride; + } + + public double getScale_factor() { + return scale_factor; + } + + public void setScale_factor(double scale_factor) { + this.scale_factor = scale_factor; + } + + public String getUnits() { + return units; + } + + public void setUnits(String units) { + this.units = units; + } + + public String getFlag_values() { + return flag_values; + } + + public void setFlag_values(String flag_values) { + this.flag_values = flag_values; + } + + public String getFlag_meanings() { + return flag_meanings; + } + + public void setFlag_meanings(String flag_meanings) { + this.flag_meanings = flag_meanings; + } + + public String getData_layout() { + return data_layout; + } + + public void setData_layout(String data_layout) { + this.data_layout = data_layout; + } + + public String getStandard_name() { + return standard_name; + } + + public void setStandard_name(String standard_name) { + this.standard_name = standard_name; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableRegistry.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableRegistry.java new file mode 100644 index 000000000..b3fb67c47 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableRegistry.java @@ -0,0 +1,97 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +public class VariableRegistry { + + private Map variables; + + public static VariableRegistry load(String resourceKey) { + ObjectMapper mapper = new ObjectMapper(); + String path = resourceKey + "/variables.json"; + try (InputStream in = VariableRegistry.class.getResourceAsStream(path)) { + if (in == null) { + throw new RuntimeException("Resource not found: " + path); + } + final VariableRegistry variableRegistry = mapper.readValue(in, VariableRegistry.class); + variableRegistry.initialize(); + return variableRegistry; + } catch (IOException e) { + throw new RuntimeException("Failed to load registry from: " + path, e); + } + } + + public void clear() { + if (variables != null) { + variables.clear(); + } + } + + private void initialize() { + final ArrayList toBeDeleted = new ArrayList<>(); + final HashMap toBeAdded = new HashMap<>(); + + for (final String key: variables.keySet()) { + if (key.contains("*")) { + // expand to bands wise reading variable + // and mark to be removed + final VariableDefinition variableDefinition = variables.get(key); + toBeAdded.putAll(expandWildcardVariable(key, variableDefinition)); + toBeDeleted.add(key); + } + } + + for (String key: toBeDeleted) { + variables.remove(key); + } + variables.putAll(toBeAdded); + } + + private Map expandWildcardVariable(String key, VariableDefinition variableDefinition) { + final Map expanded = new HashMap<>(); + final int stride = variableDefinition.getStride(); + final int initialOffset = variableDefinition.getOffset(); + + for (int offset = 0; offset < stride; offset++) { + final VariableDefinition layerVariableDefinition = new VariableDefinition(); + layerVariableDefinition.setOffset(initialOffset + 4 * offset); + layerVariableDefinition.setStride(stride); + layerVariableDefinition.setScale_factor(variableDefinition.getScale_factor()); + layerVariableDefinition.setData_type(variableDefinition.getData_type()); + layerVariableDefinition.setUnits(variableDefinition.getUnits()); + layerVariableDefinition.setStandard_name(variableDefinition.getStandard_name()); + + final String layerKey = buildLayerKey(key, offset); + expanded.put(layerKey, layerVariableDefinition); + } + + return expanded; + } + + private String buildLayerKey(String key, int offset) { + String bandNumber = String.format("%02d", offset + 1); + return key.replace("*", bandNumber); + } + + public Map getVariables() { + return variables; + } + + public void setVariables(Map variables) { + this.variables = variables; + } + + public VariableDefinition getVariableDef(String variableName) { + VariableDefinition def = variables.get(variableName); + if (def == null) { + throw new IllegalArgumentException("Variable not defined: " + variableName); + } + return def; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/ASMUSA_MDR.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/ASMUSA_MDR.java new file mode 100644 index 000000000..c91fb9e05 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/ASMUSA_MDR.java @@ -0,0 +1,10 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat.record_types; + +import com.bc.fiduceo.reader.amsu_mhs.nat.GENERIC_RECORD_HEADER; + +public class ASMUSA_MDR extends MDR { + + public ASMUSA_MDR(GENERIC_RECORD_HEADER header, byte[] payload) { + super(header, payload); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/GEADR.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/GEADR.java new file mode 100644 index 000000000..be55cb750 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/GEADR.java @@ -0,0 +1,11 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat.record_types; + +import com.bc.fiduceo.reader.amsu_mhs.nat.GENERIC_RECORD_HEADER; +import com.bc.fiduceo.reader.amsu_mhs.nat.Record; + +public class GEADR extends Record { + + public GEADR(GENERIC_RECORD_HEADER header, byte[] payload) { + super(header, payload); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/GIADR.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/GIADR.java new file mode 100644 index 000000000..082fe2843 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/GIADR.java @@ -0,0 +1,11 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat.record_types; + +import com.bc.fiduceo.reader.amsu_mhs.nat.GENERIC_RECORD_HEADER; +import com.bc.fiduceo.reader.amsu_mhs.nat.Record; + +public class GIADR extends Record { + + public GIADR(GENERIC_RECORD_HEADER header, byte[] payload) { + super(header, payload); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/IPR.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/IPR.java new file mode 100644 index 000000000..719fc6d98 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/IPR.java @@ -0,0 +1,11 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat.record_types; + +import com.bc.fiduceo.reader.amsu_mhs.nat.GENERIC_RECORD_HEADER; +import com.bc.fiduceo.reader.amsu_mhs.nat.Record; + +public class IPR extends Record { + + public IPR(GENERIC_RECORD_HEADER header, byte[] payload) { + super(header, payload); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/MDR.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/MDR.java new file mode 100644 index 000000000..0ff510d44 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/MDR.java @@ -0,0 +1,38 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat.record_types; + +import com.bc.fiduceo.reader.amsu_mhs.nat.GENERIC_RECORD_HEADER; +import com.bc.fiduceo.reader.amsu_mhs.nat.Record; +import org.apache.commons.lang3.Conversion; + +public class MDR extends Record { + + public MDR(GENERIC_RECORD_HEADER header, byte[] payload) { + super(header, payload); + } + + public int[] parseVariable(String variableName) { + byte[] payload = getPayload(); + int offset = 0; + int count = 0; + int stride = 0; + int size = 0; + if (variableName.equals("latitude")) { + offset = 2082; + count = 30; + stride = 2; + size = 4; + } else if (variableName.equals("longitude")) { + offset = 2086; + count = 30; + stride = 2; + size = 4; + } + + + int[] variableRawData = new int[count]; + for (int i = 0; i < count; i++) { + variableRawData[i] = Conversion.byteArrayToInt(payload, offset + stride * i * size, variableRawData[i], 0, size); + } + return variableRawData; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/MPHR.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/MPHR.java new file mode 100644 index 000000000..a8b4ff742 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/MPHR.java @@ -0,0 +1,84 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat.record_types; + +import com.bc.fiduceo.reader.amsu_mhs.nat.GENERIC_RECORD_HEADER; +import com.bc.fiduceo.reader.amsu_mhs.nat.Record; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.TimeZone; + +import static com.bc.fiduceo.reader.amsu_mhs.nat.EPS_Constants.SENSING_START_KEY; +import static com.bc.fiduceo.reader.amsu_mhs.nat.EPS_Constants.SENSING_STOP_KEY; + +public class MPHR extends Record { + + public static final int PRODUCT_NAME_OFFSET = 20; + public static final int STRING_SIZE = 100; + private final HashMap attributesMap; + + public MPHR(GENERIC_RECORD_HEADER header, byte[] payload) { + super(header, payload); + attributesMap = new HashMap<>(); + } + + public Date getDate(String key) throws IOException { + String attributeValue = attributesMap.get(key); + if (attributeValue == null) { + int offset; + if (key.equals(SENSING_START_KEY)) { + offset = 700; + } else if (key.equals(SENSING_STOP_KEY)) { + offset = 748; + } else { + throw new IllegalStateException("Unknown attribute key: " + key); + } + attributeValue = extractStringAttribute(offset, 48); + attributesMap.put(key, attributeValue); + } + + return parseDate(attributeValue); + } + + public Date getSensingStart() throws IOException { + return getDate(SENSING_START_KEY); + } + + public Date getSensingStop() throws IOException { + return getDate(SENSING_STOP_KEY); + } + + + private Date parseDate(String date) throws IOException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmssX"); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + try { + return sdf.parse(date); + } catch (ParseException e) { + throw new IOException("Could not parse time: " + date, e); + } + } + + public String getProductName() { + String productName = attributesMap.get("PRODUCT_NAME"); + if (productName == null) { + productName = extractStringAttribute(PRODUCT_NAME_OFFSET, STRING_SIZE); + attributesMap.put("PRODUCT_NAME", productName); + } + return productName; + } + + private String extractStringAttribute(int offset, int size) { + final byte[] nameBuffer = new byte[100]; + System.arraycopy(getPayload(), offset, nameBuffer, 0, size); + final String attributeString = new String(nameBuffer, StandardCharsets.US_ASCII).trim(); + final String[] parts = attributeString.split("="); + if (parts.length != 2) { + throw new IllegalStateException("Invalid attribute formatting: " + attributeString); + } + return parts[1].trim(); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/SPHR.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/SPHR.java new file mode 100644 index 000000000..92793d266 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/SPHR.java @@ -0,0 +1,11 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat.record_types; + +import com.bc.fiduceo.reader.amsu_mhs.nat.GENERIC_RECORD_HEADER; +import com.bc.fiduceo.reader.amsu_mhs.nat.Record; + +public class SPHR extends Record { + + public SPHR(GENERIC_RECORD_HEADER header, byte[] payload) { + super(header, payload); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/VEADR.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/VEADR.java new file mode 100644 index 000000000..81a609cd0 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/VEADR.java @@ -0,0 +1,11 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat.record_types; + +import com.bc.fiduceo.reader.amsu_mhs.nat.GENERIC_RECORD_HEADER; +import com.bc.fiduceo.reader.amsu_mhs.nat.Record; + +public class VEADR extends Record { + + public VEADR(GENERIC_RECORD_HEADER header, byte[] payload) { + super(header, payload); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/VIADR.java b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/VIADR.java new file mode 100644 index 000000000..0eae1df47 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/VIADR.java @@ -0,0 +1,11 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat.record_types; + +import com.bc.fiduceo.reader.amsu_mhs.nat.GENERIC_RECORD_HEADER; +import com.bc.fiduceo.reader.amsu_mhs.nat.Record; + +public class VIADR extends Record { + + public VIADR(GENERIC_RECORD_HEADER header, byte[] payload) { + super(header, payload); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/avhrr_frac/AVHRR_FRAC_Reader.java b/core/src/main/java/com/bc/fiduceo/reader/avhrr_frac/AVHRR_FRAC_Reader.java index ad91956cd..7a2bca870 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/avhrr_frac/AVHRR_FRAC_Reader.java +++ b/core/src/main/java/com/bc/fiduceo/reader/avhrr_frac/AVHRR_FRAC_Reader.java @@ -7,9 +7,10 @@ import com.bc.fiduceo.reader.ReaderContext; import com.bc.fiduceo.reader.ReaderUtils; import com.bc.fiduceo.reader.snap.SNAP_Reader; +import com.bc.fiduceo.reader.time.TimeLocator_StartStopDate; import com.bc.fiduceo.reader.time.TimeLocator; import com.bc.fiduceo.util.NetCDFUtils; -import org.esa.s3tbx.dataio.avhrr.AvhrrConstants; +import eu.esa.opt.dataio.avhrr.AvhrrConstants; import org.esa.snap.core.datamodel.ProductData; import org.esa.snap.core.datamodel.RasterDataNode; import org.esa.snap.core.datamodel.VirtualBand; @@ -75,7 +76,7 @@ public PixelLocator getSubScenePixelLocator(Polygon sceneGeometry) throws IOExce public TimeLocator getTimeLocator() { final ProductData.UTC startTime = product.getStartTime(); final ProductData.UTC endTime = product.getEndTime(); - return new AVHRR_FRAC_TimeLocator(startTime.getAsDate(), endTime.getAsDate(), product.getSceneRasterHeight()); + return new TimeLocator_StartStopDate(startTime.getAsDate(), endTime.getAsDate(), product.getSceneRasterHeight()); } @Override diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/InsituReader.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/InsituReader.java new file mode 100644 index 000000000..c5e8a16b2 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/InsituReader.java @@ -0,0 +1,12 @@ +package com.bc.fiduceo.reader.insitu; + +import com.bc.fiduceo.reader.netcdf.NetCDFReader; +import ucar.ma2.Array; + +import java.io.IOException; + +public abstract class InsituReader extends NetCDFReader { + + abstract public Array getSourceArray(String variableName) throws IOException; + +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/CsvFormatConfig.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/CsvFormatConfig.java new file mode 100644 index 000000000..ae429fc91 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/CsvFormatConfig.java @@ -0,0 +1,165 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import org.esa.snap.core.datamodel.ProductData; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +public class CsvFormatConfig { + + private String name; + private String delimiter; + private char commentChar; + private String regex; + private String longitudeName; + private String latitudeName; + private String timeName = "time"; + private String timeFormat; + private List timeVars; + private boolean locationFromStationDatabase; + private List variables; + private StationDatabase stationDatabase; + + public static CsvFormatConfig loadConfig(String resourceKey) { + ObjectMapper mapper = new ObjectMapper(); + mapper.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE); + String path = resourceKey + "_config.json"; + + try (InputStream in = CsvFormatConfig.class.getResourceAsStream(path)) { + if (in == null) { + throw new RuntimeException("Resource not found: " + path); + } + + return mapper.readValue(in, CsvFormatConfig.class); + } catch (IOException e) { + throw new RuntimeException("Failed to load config from: " + path, e); + } + } + + public List getAllVariables() { + List vars = new ArrayList<>(); + + if (stationDatabase != null) { + vars.addAll(stationDatabase.getVariables()); + } + + if (hasTimeVars()) { + GenericVariable time = new GenericVariable(timeName, 'v',"int", ProductData.TYPE_INT64, -1.0, "seconds since 1970-01-01", null, "time", null); + vars.add(time); + + for (GenericVariable var : variables) { + if (!timeVars.contains(var.getName())) { + vars.add(var); + } + } + } else { + vars.addAll(variables); + } + + return vars; + } + + private boolean hasTimeVars() { + return timeVars != null && !timeVars.isEmpty(); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDelimiter() { + return delimiter; + } + + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; + } + + public char getCommentChar() { + return commentChar; + } + + public void setCommentChar(char commentChar) { + this.commentChar = commentChar; + } + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } + + public String getLongitudeName() { + return longitudeName; + } + + public void setLongitudeName(String longitudeName) { + this.longitudeName = longitudeName; + } + + public String getLatitudeName() { + return latitudeName; + } + + public void setLatitudeName(String latitudeName) { + this.latitudeName = latitudeName; + } + + public String getTimeName() { + return timeName; + } + + public void setTimeName(String timeName) { + this.timeName = timeName; + } + + public String getTimeFormat() { + return timeFormat; + } + + public void setTimeFormat(String timeFormat) { + this.timeFormat = timeFormat; + } + + public List getTimeVars() { + return timeVars; + } + + public void setTimeVars(List timeVars) { + this.timeVars = timeVars; + } + + public boolean isLocationFromStationDatabase() { + return locationFromStationDatabase; + } + + public void setLocationFromStationDatabase(boolean locationFromStationDatabase) { + this.locationFromStationDatabase = locationFromStationDatabase; + } + + public List getVariables() { + return variables; + } + + public void setVariables(List variables) { + this.variables = variables; + } + + public StationDatabase getStationDatabase() { + return stationDatabase; + } + + public void setStationDatabase(StationDatabase stationDatabase) { + this.stationDatabase = stationDatabase; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GbovReaderPlugin.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GbovReaderPlugin.java new file mode 100644 index 000000000..4d30942eb --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GbovReaderPlugin.java @@ -0,0 +1,27 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.ReaderContext; +import com.bc.fiduceo.reader.ReaderPlugin; + + +public class GbovReaderPlugin implements ReaderPlugin { + + private final String[] SUPPORTED_KEYS = {"gbov"}; + + @Override + public Reader createReader(ReaderContext readerContext) { + return new GenericCsvReader(GenericCsvHelper.RESOURCE_KEY_GBOV); + } + + @Override + public String[] getSupportedSensorKeys() { + return SUPPORTED_KEYS; + } + + @Override + public DataType getDataType() { + return DataType.INSITU; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvHelper.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvHelper.java new file mode 100644 index 000000000..16c049880 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvHelper.java @@ -0,0 +1,251 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import com.bc.fiduceo.util.TimeUtils; +import ucar.ma2.DataType; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.List; + +public class GenericCsvHelper { + + public static final String RESOURCE_KEY_NDBC_SM = "NDBC_SM"; + public static final String RESOURCE_KEY_NDBC_CW = "NDBC_CW"; + public static final String RESOURCE_KEY_GBOV = "GBOV"; + + + public static int[] extractYearMonthDayFromFilename(String filename, String resourceKey) { + int[] ymd = new int[3]; + if (resourceKey.equals(GenericCsvHelper.RESOURCE_KEY_NDBC_SM) || resourceKey.equals(GenericCsvHelper.RESOURCE_KEY_NDBC_CW)) { + final int dotIndex = filename.indexOf('.'); + final String yearString = filename.substring(dotIndex - 4, dotIndex); + ymd[0] = Integer.parseInt(yearString); + ymd[1] = 1; + ymd[2] = 1; + + return ymd; + } else if (resourceKey.equals(GenericCsvHelper.RESOURCE_KEY_GBOV)) { + final String[] fileSplit = filename.split("__"); + final String startDate = fileSplit[3]; + final String yearString = startDate.substring(0, 4); + final String monthString = startDate.substring(4, 6); + final String dayString = startDate.substring(6, 8); + ymd[0] = Integer.parseInt(yearString); + ymd[1] = Integer.parseInt(monthString); + ymd[2] = Integer.parseInt(dayString); + + return ymd; + } + throw new IllegalStateException("Unsupported format for file: '" + filename + "' and resourceKey '" + resourceKey + "'."); + } + + public static List parseData(File file, CsvFormatConfig config, String resourceKey) throws IOException { + List records = new ArrayList<>(); + + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == config.getCommentChar()) { + continue; + } + + String[] tokens = tokenize(line, config.getDelimiter()); + List vars = config.getVariables(); + + if (tokens.length < vars.size()) continue; + + GenericRecord record = parseLine(tokens, vars, resourceKey, config); + records.add(record); + } + } + + return records; + } + + private static String[] tokenize(String line, String delimiter) { + if ("space".equalsIgnoreCase(delimiter)) { + return line.trim().split("\\s+"); + } + return line.split(delimiter); + } + + private static GenericRecord parseLine(String[] tokens, List vars, String resourceKey, CsvFormatConfig config) { + GenericRecord record = new GenericRecord(); + + if (resourceKey.equals(GenericCsvHelper.RESOURCE_KEY_NDBC_CW) || resourceKey.equals(GenericCsvHelper.RESOURCE_KEY_NDBC_SM)) { + GenericVariable yyVar = null, mmVar = null, ddVar = null, hhVar = null, minVar = null; + int yyIndex = -1, mmIndex = -1, ddIndex = -1, hhIndex = -1, minIndex = -1; + + for (int ii = 0; ii < vars.size(); ii++) { + String name = vars.get(ii).getName(); + switch (name) { + case "YY": yyVar = vars.get(ii); yyIndex = ii; break; + case "MM": mmVar = vars.get(ii); mmIndex = ii; break; + case "DD": ddVar = vars.get(ii); ddIndex = ii; break; + case "hh": hhVar = vars.get(ii); hhIndex = ii; break; + case "mm": minVar = vars.get(ii); minIndex = ii; break; + } + } + + if (yyIndex >= 0 && mmIndex >= 0 && ddIndex >= 0 && hhIndex >= 0 && minIndex >= 0) { + int year = ((Number) parseValue(tokens[yyIndex], yyVar.getType())).intValue(); + int month = ((Number) parseValue(tokens[mmIndex], mmVar.getType())).intValue(); + int day = ((Number) parseValue(tokens[ddIndex], ddVar.getType())).intValue(); + int hour = ((Number) parseValue(tokens[hhIndex], hhVar.getType())).intValue(); + int minute = ((Number) parseValue(tokens[minIndex], minVar.getType())).intValue(); + + final Calendar calendar = TimeUtils.getUTCCalendar(); + calendar.setTimeInMillis(0); + calendar.set(Calendar.YEAR, year); + calendar.set(Calendar.MONTH, month - 1); // calendar wants month zero-based + calendar.set(Calendar.DAY_OF_MONTH, day); + calendar.set(Calendar.HOUR_OF_DAY, hour); + calendar.set(Calendar.MINUTE, minute); + int seconds = (int) (calendar.getTimeInMillis() * 0.001); + + record.put(config.getTimeName(), seconds); + } + + for (int ii = 0; ii < vars.size(); ii++) { + String name = vars.get(ii).getName(); + if (Arrays.asList("YY", "MM", "DD", "hh", "mm").contains(name)) continue; + record.put(name, parseValue(tokens[ii], vars.get(ii).getType())); + } + } else { + for (int ii = 0; ii < vars.size(); ii++) { + GenericVariable var = vars.get(ii); + + if (var.getName().equals(config.getTimeName())) { + String value = (String) parseValue(tokens[ii], "string"); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(config.getTimeFormat()).withZone(ZoneOffset.UTC); + Instant instant = Instant.from(formatter.parse(value)); + record.put(config.getTimeName(), (int) instant.getEpochSecond()); + continue; + } + + record.put(var.getName(), parseValue(tokens[ii], var.getType())); + } + } + + return record; + } + + public static Object parseValue(String token, String type) { + if (token == null || token.isEmpty()) { + switch (type) { + case "float": + return Float.NaN; + case "double": + return Double.NaN; + default: + return null; + } + } + + token = token.trim(); + + switch (type) { + case "byte": + return Byte.parseByte(token); + case "short": + return Short.parseShort(token); + case "int": + return Integer.parseInt(token); + case "long": + return Long.parseLong(token); + case "float": + return Float.parseFloat(token); + case "double": + return Double.parseDouble(token); + default: + return token; + } + } + + public static String getPrimaryIdFromFilename(File file, String resourceKey) { + String filename = file.getName(); + if (resourceKey.equals(RESOURCE_KEY_NDBC_SM) || resourceKey.equals(RESOURCE_KEY_NDBC_CW)) { + if (filename.length() >= 5) { + return file.getName().substring(0,5); + } + } + if (resourceKey.equals(RESOURCE_KEY_GBOV)) { + String[] split = filename.split("__"); + return split[1].replace("--", " "); + } + throw new IllegalArgumentException("Unsupported format for file: " + file.getAbsolutePath()); + } + + public static String getSecondaryIdFromFilename(File file, String resourceKey) { + String filename = file.getName(); + if (resourceKey != null && resourceKey.equals(RESOURCE_KEY_GBOV)) { + String[] split = filename.split("__"); + return split[2].replace("--", " "); + } + return null; + } + + public static DataType getNcDataType(String type) { + switch (type) { + case "byte": + return DataType.BYTE; + case "short": + return DataType.SHORT; + case "int": + return DataType.INT; + case "long": + return DataType.LONG; + case "float": + return DataType.FLOAT; + case "double": + return DataType.DOUBLE; + default: + return DataType.STRING; + } + } + + public static Class getFillValueClass(String type) { + switch (type) { + case "byte": + return byte.class; + case "short": + return short.class; + case "int": + return int.class; + case "long": + return long.class; + case "float": + return float.class; + case "double": + return double.class; + default: + throw new RuntimeException("not implemented for type " + type); + } + } + + public static Number castFillValue(Double fillValue, String type) { + switch (type.toLowerCase()) { + case "byte": + return fillValue.byteValue(); + case "short": + return fillValue.shortValue(); + case "int": + return fillValue.intValue(); + case "long": + return fillValue.longValue(); + case "float": + return fillValue.floatValue(); + case "double": + default: + return fillValue; + } + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader.java new file mode 100644 index 000000000..40523bd5f --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader.java @@ -0,0 +1,276 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.core.NodeType; +import com.bc.fiduceo.geometry.Polygon; +import com.bc.fiduceo.location.PixelLocator; +import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.netcdf.StringVariable; +import com.bc.fiduceo.reader.time.TimeLocator; +import com.bc.fiduceo.reader.time.TimeLocator_MillisSince1970; +import com.bc.fiduceo.util.NetCDFUtils; +import com.bc.fiduceo.util.VariableProxy; +import ucar.ma2.Array; +import ucar.ma2.ArrayInt; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * Generic CSV/ASCII reader driven by an json config file. + */ +public class GenericCsvReader implements Reader { + + private String resourceKey; + + private CsvFormatConfig config; + private List variables; + private List records; + private GenericRecord stationDatabaseRecord; + + public GenericCsvReader(String resourceKey) { + this.resourceKey = resourceKey; + } + + @Override + public void open(File file) throws IOException { + config = CsvFormatConfig.loadConfig(resourceKey); + records = GenericCsvHelper.parseData(file, config, resourceKey); + variables = config.getAllVariables(); + + StationDatabase stationDatabase = config.getStationDatabase(); + if (stationDatabase != null) { + String primaryId = GenericCsvHelper.getPrimaryIdFromFilename(file, resourceKey); + String secondaryId = GenericCsvHelper.getSecondaryIdFromFilename(file, resourceKey); + stationDatabaseRecord = stationDatabase.extractRecord(primaryId, secondaryId); + } + } + + @Override + public void close() throws IOException { + if (records != null) { + for (GenericRecord record : records) { + record.getValues().clear(); + } + records.clear(); + records = null; + } + if (variables != null) { + variables.clear(); + variables = null; + } + if (stationDatabaseRecord != null) { + stationDatabaseRecord.getValues().clear(); + stationDatabaseRecord = null; + } + if (config != null) { + config.getVariables().clear(); + final StationDatabase stationDatabase = config.getStationDatabase(); + if (stationDatabase != null) { + stationDatabase.getStations().clear(); + stationDatabase.getVariables().clear(); + } + config = null; + } + } + + @Override + public AcquisitionInfo read() throws IOException { + AcquisitionInfo acquisitionInfo = new AcquisitionInfo(); + + int minTime = Integer.MAX_VALUE; + int maxTime = Integer.MIN_VALUE; + String timeName = config.getTimeName(); + for (final GenericRecord record : records) { + int time = (int) record.getValues().get(timeName); + if (time < minTime) { + minTime = time; + } + if (time > maxTime) { + maxTime = time; + } + } + + acquisitionInfo.setSensingStart(new Date(minTime * 1000L)); + acquisitionInfo.setSensingStop(new Date(maxTime * 1000L)); + + acquisitionInfo.setNodeType(NodeType.UNDEFINED); + + return acquisitionInfo; + } + + @Override + public String getRegEx() { + if (config == null) { + config = CsvFormatConfig.loadConfig(resourceKey); + } + return config.getRegex(); + } + + @Override + public PixelLocator getPixelLocator() throws IOException { + throw new RuntimeException("not implemented"); + } + + @Override + public PixelLocator getSubScenePixelLocator(Polygon sceneGeometry) throws IOException { + throw new RuntimeException("not implemented"); + } + + @Override + public TimeLocator getTimeLocator() throws IOException { + long[] timeArray = new long[records.size()]; + + int ii = 0; + for (final GenericRecord record : records) { + String timeName = config.getTimeName(); + int timeInSeconds = (int) record.getValues().get(timeName); + timeArray[ii] = timeInSeconds * 1000L; + ii++; + } + + return new TimeLocator_MillisSince1970(timeArray); + } + + @Override + public int[] extractYearMonthDayFromFilename(String fileName) { + return GenericCsvHelper.extractYearMonthDayFromFilename(fileName, resourceKey); + } + + @Override + public Array readRaw(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException { + GenericVariable variable = findVariable(variableName); + if (variable != null) { + Object o; + if (variable.getOrigin() == 's') { + o = stationDatabaseRecord.get(variableName); + } else { + GenericRecord record = records.get(centerY); + o = record.get(variableName); + } + return createResultArray(o, variable.getFillValue(), variable.getType(), interval); + } + + return null; + } + + @Override + public Array readScaled(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException { + return readRaw(centerX, centerY, interval, variableName); + } + + @Override + public ArrayInt.D2 readAcquisitionTime(int x, int y, Interval interval) throws IOException, InvalidRangeException { + String timeName = config.getTimeName(); + final Array timeArray = readRaw(x, y, interval, timeName); + + return (ArrayInt.D2) timeArray; + } + + @Override + public List getVariables() throws InvalidRangeException, IOException { + final ArrayList variables = new ArrayList<>(); + List attributes; + + for (GenericVariable var : this.variables) { + attributes = new ArrayList<>(); + setNcAttributes(attributes, var); + + DataType ncDataType = GenericCsvHelper.getNcDataType(var.getType()); + VariableProxy varProxy = new VariableProxy(var.getName(), ncDataType, attributes); + + if (ncDataType == DataType.STRING) { + StringVariable stringVar = new StringVariable(varProxy, 50); + variables.add(stringVar); + } else { + variables.add(varProxy); + } + } + + return variables; + } + + @Override + public Dimension getProductSize() throws IOException { + return new Dimension("product_size", 1, records.size()); + } + + @Override + public String getLongitudeVariableName() { + return config.getLongitudeName(); + } + + @Override + public String getLatitudeVariableName() { + return config.getLatitudeName(); + } + + private GenericVariable findVariable(String variableName) { + for (GenericVariable variable : variables) { + if (variable.getName().equals(variableName)) { + return variable; + } + } + return null; + } + + protected Array createResultArray(Object value, Number fillValue, String dataType, Interval interval) { + if (value instanceof String) { + final Array resultArray = Array.factory(DataType.STRING, new int[]{1, 1}); + resultArray.setObject(0, value); + return resultArray; + } + + if (fillValue == null) { + fillValue = NetCDFUtils.getDefaultFillValue(GenericCsvHelper.getFillValueClass(dataType)); + } + + DataType ncDataType = GenericCsvHelper.getNcDataType(dataType); + + final int windowHeight = interval.getY(); + final int windowWidth = interval.getX(); + final Array windowArray = NetCDFUtils.create(ncDataType, + new int[]{windowHeight, windowWidth}, + fillValue); + + final int windowCenterX = windowWidth / 2; + final int windowCenterY = windowHeight / 2; + + Object safeValue = value != null ? value : fillValue; + windowArray.setObject(windowWidth * windowCenterY + windowCenterX, safeValue); + return windowArray; + } + + private void setNcAttributes(List attributes, GenericVariable var) { + if (var.getUnits() != null) { + attributes.add(new Attribute(NetCDFUtils.CF_UNITS_NAME, var.getUnits())); + } + if (var.getFillValue() != null) { + Number typedFillValue = GenericCsvHelper.castFillValue(var.getFillValue(), var.getType()); + attributes.add(new Attribute(NetCDFUtils.CF_FILL_VALUE_NAME, typedFillValue)); + } else { + if (!var.getType().equals("string")) { + Number fillValue = NetCDFUtils.getDefaultFillValue(GenericCsvHelper.getFillValueClass(var.getType())); + attributes.add(new Attribute(NetCDFUtils.CF_FILL_VALUE_NAME, fillValue)); + } + } + if (var.getCfStandard() != null) { + attributes.add(new Attribute(NetCDFUtils.CF_STANDARD_NAME, var.getCfStandard())); + } + if (var.getLongName() != null) { + attributes.add(new Attribute(NetCDFUtils.CF_LONG_NAME, var.getLongName())); + } + if (var.getAncillaryVariables() != null) { + attributes.add(new Attribute(NetCDFUtils.CF_ANCILLARY_VARIABLES_NAME, var.getAncillaryVariables())); + } + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericRecord.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericRecord.java new file mode 100644 index 000000000..644ef8f48 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericRecord.java @@ -0,0 +1,21 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class GenericRecord { + + private final Map values = new LinkedHashMap<>(); + + public void put(String variableName, Object value) { + values.put(variableName, value); + } + + public Object get(String variableName) { + return values.get(variableName); + } + + public Map getValues() { + return values; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericVariable.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericVariable.java new file mode 100644 index 000000000..ef2accf4f --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/GenericVariable.java @@ -0,0 +1,125 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import org.esa.snap.core.datamodel.ProductData; + +public class GenericVariable { + + private String name; + private char origin = 'v'; // 'v' = variable (default), 's' = station + private String type; + private int productData; + private Double fillValue; + private String units; + private String longName; + private String cfStandard; + private String ancillaryVariables; + + public GenericVariable() { + } + + public GenericVariable(String name, char origin, String type, int productData, Double fillValue, String units, String longName, String cfStandard, String ancillaryVariables) { + this.name = name; + this.origin = origin; + this.type = type; + this.productData = productData; + this.fillValue = fillValue; + this.units = units; + this.longName = longName; + this.cfStandard = cfStandard; + this.ancillaryVariables = ancillaryVariables; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public char getOrigin() { + return origin; + } + + public void setOrigin(char origin) { + this.origin = origin; + } + + public String getType() { + return type; + } + + public void setType(String type) { + setProductData(parseProductData(type)); + this.type = type; + } + + public int getProductData() { + return productData; + } + + public void setProductData(int productData) { + this.productData = productData; + } + + public Double getFillValue() { + return fillValue; + } + + public void setFillValue(Double fillValue) { + this.fillValue = fillValue; + } + + public String getUnits() { + return units; + } + + public void setUnits(String units) { + this.units = units; + } + + public String getLongName() { + return longName; + } + + public void setLongName(String longName) { + this.longName = longName; + } + + public String getCfStandard() { + return cfStandard; + } + + public void setCfStandard(String cfStandard) { + this.cfStandard = cfStandard; + } + + public String getAncillaryVariables() { + return ancillaryVariables; + } + + public void setAncillaryVariables(String ancillaryVariables) { + this.ancillaryVariables = ancillaryVariables; + } + + private static int parseProductData(String type) { + switch (type.toLowerCase()) { + case "byte": + return ProductData.TYPE_INT8; + case "short": + return ProductData.TYPE_INT16; + case "int": + return ProductData.TYPE_INT32; + case "long": + return ProductData.TYPE_INT64; + case "float": + return ProductData.TYPE_FLOAT32; + case "double": + return ProductData.TYPE_FLOAT64; + case "string": + return ProductData.TYPE_ASCII; + default: + throw new IllegalArgumentException("Unsupported type: " + type); + } + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/StationDatabase.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/StationDatabase.java new file mode 100644 index 000000000..7cce6c43f --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/generic/StationDatabase.java @@ -0,0 +1,115 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import java.util.List; + +public class StationDatabase { + + private String primaryId; + private String secondaryId; + private List variables; + private List> stations; + + + public String getPrimaryId() { + return primaryId; + } + + public void setPrimaryId(String primaryId) { + this.primaryId = primaryId; + } + + public String getSecondaryId() { + return secondaryId; + } + + public void setSecondaryId(String secondaryId) { + this.secondaryId = secondaryId; + } + + public List getVariables() { + return variables; + } + + public void setVariables(List variables) { + this.variables = variables; + } + + public List> getStations() { + return stations; + } + + public void setStations(List> stations) { + this.stations = stations; + } + + public GenericRecord extractRecord(String primaryIdValue, String secondaryIdValue) { + int primaryIndex = -1; + int secondaryIndex = -1; + for (int ii = 0; ii < variables.size(); ii++) { + final GenericVariable var = variables.get(ii); + if (var.getName().equals(primaryId)) { + primaryIndex = ii; + } + if (var.getName().equals(secondaryId)) { + secondaryIndex = ii; + } + } + + if (primaryIndex == -1) { + throw new IllegalStateException("Station database does not contain variable '" + primaryId + "'."); + } + if (secondaryId != null && secondaryIndex == -1) { + throw new IllegalStateException("Station database does not contain variable '" + secondaryId + "'."); + } + + List desiredStation = extractStation(primaryIndex, primaryIdValue, secondaryIndex, secondaryIdValue); + + if (desiredStation == null) { + throw new IllegalStateException("Station database does not contain site/station with ids '" + primaryId + "'/'" + secondaryId + "."); + } + + GenericRecord resultRecord = new GenericRecord(); + for (int ii = 0; ii < desiredStation.size(); ii++) { + GenericVariable var = variables.get(ii); + Object value = desiredStation.get(ii); + + String token = (value == null) ? null : String.valueOf(value); + Object parsedValue = GenericCsvHelper.parseValue(token, var.getType()); + + resultRecord.put(var.getName(), parsedValue); + } + + return resultRecord; + } + + private List extractStation(int primaryIndex, String primaryIdValue, int secondaryIndex, String secondaryIdValue) { + List desiredStation = null; + for (int ii = 0; ii < stations.size(); ii++) { + final List station = stations.get(ii); + boolean primaryFound = false; + boolean secondaryFound = false; + + String siteId = (String) station.get(primaryIndex); + String stationId = null; + if (secondaryId != null) { + stationId = (String) station.get(secondaryIndex); + } else { + secondaryFound = true; + } + + if (siteId != null && siteId.equalsIgnoreCase(primaryIdValue)) { + primaryFound = true; + } + if (secondaryId != null && stationId != null && stationId.equalsIgnoreCase(secondaryIdValue)) { + secondaryFound = true; + } + + if (primaryFound && secondaryFound) { + desiredStation = station; + break; + } + } + return desiredStation; + } + +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcCWReaderPlugin.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcCWReaderPlugin.java index 5c8d52a28..e540424a4 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcCWReaderPlugin.java +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcCWReaderPlugin.java @@ -4,6 +4,8 @@ import com.bc.fiduceo.reader.Reader; import com.bc.fiduceo.reader.ReaderContext; import com.bc.fiduceo.reader.ReaderPlugin; +import com.bc.fiduceo.reader.insitu.generic.GenericCsvHelper; +import com.bc.fiduceo.reader.insitu.generic.GenericCsvReader; public class NdbcCWReaderPlugin implements ReaderPlugin { @@ -11,7 +13,7 @@ public class NdbcCWReaderPlugin implements ReaderPlugin { @Override public Reader createReader(ReaderContext readerContext) { - return new NdbcCWReader(); + return new GenericCsvReader(GenericCsvHelper.RESOURCE_KEY_NDBC_CW); } @Override diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReader.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReader.java index 735136309..b08790efa 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReader.java +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReader.java @@ -35,6 +35,7 @@ class NdbcSMReader extends NdbcReader { private static final String REG_EX_SM = "\\w{5}h\\d{4}.txt"; private static final String WVHT = "WVHT"; + private static final String WTMP = "WTMP"; private static final String DPD = "DPD"; private static final String APD = "APD"; private static final String MWD = "MWD"; @@ -116,6 +117,7 @@ public TimeLocator getTimeLocator() throws IOException { @Override public Array readRaw(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException { final SmRecord record = records.get(centerY); + // @todo 2 tb/tb handle positions out of range 2023-05-02 switch (variableName) { case STATION_ID: @@ -150,6 +152,8 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl return createResultArray(record.gustSpeed, 99.f, DataType.FLOAT, interval); case WVHT: return createResultArray(record.waveHeight, 99.f, DataType.FLOAT, interval); + case WTMP: + return createResultArray(record.seaSurfTemp, 999.f, DataType.FLOAT, interval); case DPD: return createResultArray(record.domWavePeriod, 99.f, DataType.FLOAT, interval); case APD: @@ -215,6 +219,13 @@ public List getVariables() throws InvalidRangeException, IOException { createWindSpeedVariable(variables); createGustSpeedVariable(variables, "Peak 5 or 8 second gust speed (m/s) measured during the eight-minute or two-minute period. The 5 or 8 second period can be determined by payload."); + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "degC")); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, 999.f)); + attributes.add(new Attribute(CF_STANDARD_NAME, "sea_surface_temperature")); + attributes.add(new Attribute(CF_LONG_NAME, "Sea surface temperature (Celsius). For buoys the depth is referenced to the hull's waterline. For fixed platforms it varies with tide, but is referenced to, or near Mean Lower Low Water (MLLW).")); + variables.add(new VariableProxy(WTMP, DataType.FLOAT, attributes)); + attributes = new ArrayList<>(); attributes.add(new Attribute(CF_UNITS_NAME, "m")); attributes.add(new Attribute(CF_FILL_VALUE_NAME, 99.f)); diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderPlugin.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderPlugin.java index 18c5efb02..b64059e94 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderPlugin.java +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderPlugin.java @@ -4,6 +4,8 @@ import com.bc.fiduceo.reader.Reader; import com.bc.fiduceo.reader.ReaderContext; import com.bc.fiduceo.reader.ReaderPlugin; +import com.bc.fiduceo.reader.insitu.generic.GenericCsvHelper; +import com.bc.fiduceo.reader.insitu.generic.GenericCsvReader; public class NdbcSMReaderPlugin implements ReaderPlugin { @@ -11,7 +13,7 @@ public class NdbcSMReaderPlugin implements ReaderPlugin { @Override public Reader createReader(ReaderContext readerContext) { - return new NdbcSMReader(); + return new GenericCsvReader(GenericCsvHelper.RESOURCE_KEY_NDBC_SM); } @Override diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParser.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParser.java index 338b36195..100f4ef3e 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParser.java +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParser.java @@ -2,6 +2,7 @@ import com.bc.fiduceo.util.NetCDFUtils; import com.bc.fiduceo.util.VariableProxy; +import org.esa.snap.core.util.StringUtils; import ucar.ma2.DataType; import ucar.nc2.Attribute; import ucar.nc2.Variable; @@ -21,16 +22,16 @@ List getVariables() { createCommonVariables(variables); List attributes = new ArrayList<>(); - attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(byte.class))); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(float.class))); attributes.add(new Attribute(CF_STANDARD_NAME, "sea_ice_area_fraction")); attributes.add(new Attribute(CF_UNITS_NAME, "percent")); - variables.add(new VariableProxy("SIC-total", DataType.BYTE, attributes)); + variables.add(new VariableProxy("SIC-total", DataType.FLOAT, attributes)); attributes = new ArrayList<>(); - attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(byte.class))); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(float.class))); attributes.add(new Attribute(CF_STANDARD_NAME, "sea_ice_area_fraction")); attributes.add(new Attribute(CF_UNITS_NAME, "percent")); - variables.add(new VariableProxy("SIC-primary", DataType.BYTE, attributes)); + variables.add(new VariableProxy("SIC-primary", DataType.FLOAT, attributes)); attributes = new ArrayList<>(); attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(byte.class))); @@ -64,10 +65,10 @@ List getVariables() { variables.add(new VariableProxy("Snow-depth-primary", DataType.FLOAT, attributes)); attributes = new ArrayList<>(); - attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(byte.class))); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(float.class))); attributes.add(new Attribute(CF_STANDARD_NAME, "sea_ice_area_fraction")); attributes.add(new Attribute(CF_UNITS_NAME, "percent")); - variables.add(new VariableProxy("SIC-secondary", DataType.BYTE, attributes)); + variables.add(new VariableProxy("SIC-secondary", DataType.FLOAT, attributes)); attributes = new ArrayList<>(); attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(byte.class))); @@ -101,10 +102,10 @@ List getVariables() { variables.add(new VariableProxy("Snow-depth-secondary", DataType.FLOAT, attributes)); attributes = new ArrayList<>(); - attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(byte.class))); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(float.class))); attributes.add(new Attribute(CF_STANDARD_NAME, "sea_ice_area_fraction")); attributes.add(new Attribute(CF_UNITS_NAME, "percent")); - variables.add(new VariableProxy("SIC-tertiary", DataType.BYTE, attributes)); + variables.add(new VariableProxy("SIC-tertiary", DataType.FLOAT, attributes)); attributes = new ArrayList<>(); attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(byte.class))); @@ -156,9 +157,9 @@ List getVariables() { variables.add(new VariableProxy("Wind-speed", DataType.FLOAT, attributes)); attributes = new ArrayList<>(); - attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(short.class))); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(float.class))); attributes.add(new Attribute(CF_UNITS_NAME, "degree")); - variables.add(new VariableProxy("Wind-direction", DataType.SHORT, attributes)); + variables.add(new VariableProxy("Wind-direction", DataType.FLOAT, attributes)); attributes = new ArrayList<>(); attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(byte.class))); @@ -166,7 +167,7 @@ List getVariables() { attributes = new ArrayList<>(); attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(byte.class))); - variables.add(new VariableProxy("Cloud-cover", DataType.BYTE, attributes)); + variables.add(new VariableProxy("Cloud-cover", DataType.BYTE, attributes)); attributes = new ArrayList<>(); attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(byte.class))); @@ -193,22 +194,22 @@ Section parse(String[] tokens, int offset) throws ParseException { section.add("longitude", parseFloat(tokens[offset + 1])); section.add("time", parseUtcTime(tokens[offset + 2])); section.add("reference-id", parseString(tokens[offset + 3])); - section.add("SIC-total", parseByte(tokens[offset + 4])); - section.add("SIC-primary", parseByte(tokens[offset + 5])); + section.add("SIC-total", parseFloat(tokens[offset + 4])); + section.add("SIC-primary", parseFloat(tokens[offset + 5])); section.add("Ice-type-primary", parseByte(tokens[offset + 6])); section.add("SIT-primary", parseFloat(tokens[offset + 7])); section.add("Ridged-ice-fraction-primary", parseFloat(tokens[offset + 8])); section.add("Ridge-height-primary", parseFloat(tokens[offset + 9])); section.add("Snow-cover-type-primary", parseByte(tokens[offset + 10])); section.add("Snow-depth-primary", parseFloat(tokens[offset + 11])); - section.add("SIC-secondary", parseByte(tokens[offset + 12])); + section.add("SIC-secondary", parseFloat(tokens[offset + 12])); section.add("Ice-type-secondary", parseByte(tokens[offset + 13])); section.add("SIT-secondary", parseFloat(tokens[offset + 14])); section.add("Ridged-ice-fraction-secondary", parseFloat(tokens[offset + 15])); section.add("Ridge-height-secondary", parseFloat(tokens[offset + 16])); section.add("Snow-cover-type-secondary", parseByte(tokens[offset + 17])); section.add("Snow-depth-secondary", parseFloat(tokens[offset + 18])); - section.add("SIC-tertiary", parseByte(tokens[offset + 19])); + section.add("SIC-tertiary", parseFloat(tokens[offset + 19])); section.add("Ice-type-tertiary", parseByte(tokens[offset + 20])); section.add("SIT-tertiary", parseFloat(tokens[offset + 21])); section.add("Ridged-ice-fraction-tertiary", parseFloat(tokens[offset + 22])); @@ -218,7 +219,7 @@ Section parse(String[] tokens, int offset) throws ParseException { section.add("Sea-water-temperature", parseFloat(tokens[offset + 26])); section.add("Air-temperature", parseFloat(tokens[offset + 27])); section.add("Wind-speed", parseFloat(tokens[offset + 28])); - section.add("Wind-direction", parseShort(tokens[offset + 29])); + section.add("Wind-direction", parseFloat(tokens[offset + 29])); section.add("Visibility", parseByte(tokens[offset + 30])); section.add("Cloud-cover", parseByte(tokens[offset + 31])); section.add("Weather", parseByte(tokens[offset + 32])); diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/AbstractSectionParser.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/AbstractSectionParser.java index 2f5e49075..1dcbcb4f3 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/AbstractSectionParser.java +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/AbstractSectionParser.java @@ -34,17 +34,17 @@ static void createCommonVariables(List variables) { static void createCommonVariables(List variables, String prefix) { List attributes = new ArrayList<>(); - attributes.add(new Attribute(CF_UNITS_NAME, "degree_east")); - attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(float.class))); - attributes.add(new Attribute(CF_STANDARD_NAME, "longitude")); - variables.add(new VariableProxy(prefix + "longitude", DataType.FLOAT, attributes)); - - attributes = new ArrayList<>(); attributes.add(new Attribute(CF_UNITS_NAME, "degree_north")); attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(float.class))); attributes.add(new Attribute(CF_STANDARD_NAME, "latitude")); variables.add(new VariableProxy(prefix + "latitude", DataType.FLOAT, attributes)); + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "degree_east")); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(float.class))); + attributes.add(new Attribute(CF_STANDARD_NAME, "longitude")); + variables.add(new VariableProxy(prefix + "longitude", DataType.FLOAT, attributes)); + attributes = new ArrayList<>(); attributes.add(new Attribute(CF_UNITS_NAME, "seconds since 1970-01-01")); attributes.add(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(int.class))); diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsGtmba2InsituReaderPlugin.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsGtmba2InsituReaderPlugin.java new file mode 100644 index 000000000..9e34968de --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsGtmba2InsituReaderPlugin.java @@ -0,0 +1,8 @@ +package com.bc.fiduceo.reader.insitu.sirds_sst; + +public class SirdsGtmba2InsituReaderPlugin extends SirdsInsituReaderPlugin { + + public SirdsGtmba2InsituReaderPlugin() { + super("gtmba2-sirds"); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReader.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReader.java index 84b9d6094..a91c7ca90 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReader.java +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReader.java @@ -6,6 +6,7 @@ import com.bc.fiduceo.geometry.Polygon; import com.bc.fiduceo.location.PixelLocator; import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.insitu.InsituReader; import com.bc.fiduceo.reader.insitu.UniqueIdVariable; import com.bc.fiduceo.reader.netcdf.NetCDFReader; import com.bc.fiduceo.reader.netcdf.StringVariable; @@ -19,6 +20,7 @@ import ucar.nc2.Variable; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Calendar; import java.util.List; @@ -26,7 +28,7 @@ import static com.bc.fiduceo.reader.insitu.InsituUtils.getResultArray; -public class SirdsInsituReader extends NetCDFReader { +public class SirdsInsituReader extends InsituReader { private static final String REGEX_1 = "SSTCCI2_refdata_"; private static final String REGEX_2 = "_\\d{6}.nc"; @@ -105,6 +107,11 @@ public TimeLocator getTimeLocator() throws IOException { return (x, y) -> sensingTimes.getLong(y) * 1000L; } + @Override + public Array getSourceArray(String variableName) throws IOException { + return arrayCache.get(variableName); + } + @Override public int[] extractYearMonthDayFromFilename(String fileName) { final int lastIndex = fileName.lastIndexOf("_") + 1; diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sst_cci/SSTInsituReader.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sst_cci/SSTInsituReader.java index 95a3a3552..e45593609 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/insitu/sst_cci/SSTInsituReader.java +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/sst_cci/SSTInsituReader.java @@ -25,6 +25,7 @@ import com.bc.fiduceo.geometry.Polygon; import com.bc.fiduceo.location.PixelLocator; import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.insitu.InsituReader; import com.bc.fiduceo.reader.insitu.UniqueIdVariable; import com.bc.fiduceo.reader.netcdf.NetCDFReader; import com.bc.fiduceo.reader.time.TimeLocator; @@ -48,7 +49,7 @@ import static com.bc.fiduceo.util.TimeUtils.millisSince1978; import static com.bc.fiduceo.util.TimeUtils.secondsSince1978; -public class SSTInsituReader extends NetCDFReader { +public class SSTInsituReader extends InsituReader { private final Map fillValueMap = new HashMap<>(); private final Map arrayMap = new HashMap<>(); diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/TaoReader.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/TaoReader.java new file mode 100644 index 000000000..332e178f2 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/TaoReader.java @@ -0,0 +1,350 @@ +package com.bc.fiduceo.reader.insitu.tao; + +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.core.NodeType; +import com.bc.fiduceo.geometry.Polygon; +import com.bc.fiduceo.location.PixelLocator; +import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.netcdf.StringVariable; +import com.bc.fiduceo.reader.time.TimeLocator; +import com.bc.fiduceo.reader.time.TimeLocator_MillisSince1970; +import com.bc.fiduceo.util.NetCDFUtils; +import com.bc.fiduceo.util.VariableProxy; +import org.esa.snap.core.util.StringUtils; +import ucar.ma2.Array; +import ucar.ma2.ArrayInt; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static com.bc.fiduceo.util.NetCDFUtils.*; + +class TaoReader implements Reader { + + private final static String REG_EX = "(?:TAO|TRITON|RAMA|PIRATA)_\\w+_\\w+(-\\w+)??\\d{4}-\\d{2}.txt"; + private static final String TIME = "time"; + private static final String LONGITUDE = "longitude"; + private static final String LATITUDE = "latitude"; + private static final String SSS = "SSS"; + private static final float SSS_FILL = -9.999f; + private static final String SST = "SST"; + private static final float SST_FILL = -9.999f; + private static final String AIRT = "AIRT"; + private static final double AIRT_FILL = -9.99; + private static final String RH = "RH"; + private static final double RH_FILL = -9.99; + private static final String WSPD = "WSPD"; + private static final double WSPD_FILL = -99.9; + private static final String WDIR = "WDIR"; + private static final double WDIR_FILL = -99.9; + private static final String BARO = "BARO"; + private static final double BARO_FILL = -9.9; + private static final String RAIN = "RAIN"; + private static final double RAIN_FILL = -9.99; + private static final String Q = "Q"; + private static final String M = "M"; + + private ArrayList records; + private TimeLocator timeLocator; + + static TaoRecord parseLine(String line) { + line = line.replaceAll(" +", " "); // ensure that we only have single blanks as separator tb 2023-04-28 + final String[] tokens = StringUtils.split(line, new char[]{' '}, true); + + final TaoRecord record = new TaoRecord(); + record.time = Integer.parseInt(tokens[0]); + record.longitude = Float.parseFloat(tokens[1]); + record.latitude = Float.parseFloat(tokens[2]); + record.SSS = Float.parseFloat(tokens[3]); + record.SST = Float.parseFloat(tokens[4]); + record.AIRT = Float.parseFloat(tokens[5]); + record.RH = Float.parseFloat(tokens[6]); + record.WSPD = Float.parseFloat(tokens[7]); + record.WDIR = Float.parseFloat(tokens[8]); + record.BARO = Float.parseFloat(tokens[9]); + record.RAIN = Float.parseFloat(tokens[10]); + record.Q = Integer.parseInt(tokens[11]); + record.M = tokens[12]; + + return record; + } + + @Override + public void open(File file) throws IOException { + try (final FileReader fileReader = new FileReader(file)) { + records = new ArrayList<>(); + + final BufferedReader bufferedReader = new BufferedReader(fileReader); + String line; + while ((line = bufferedReader.readLine()) != null) { + if (line.startsWith("#")) { + // skip comment lines tb 2023-04-28 + continue; + } + + final TaoRecord record = parseLine(line); + records.add(record); + } + } + } + + @Override + public void close() throws IOException { + if (records != null) { + records.clear(); + records = null; + } + timeLocator = null; + } + + @Override + public AcquisitionInfo read() throws IOException { + final AcquisitionInfo acquisitionInfo = new AcquisitionInfo(); + int minTime = Integer.MAX_VALUE; + int maxTime = Integer.MIN_VALUE; + for (final TaoRecord record : records) { + if (record.time < minTime) { + minTime = record.time; + } + if (record.time > maxTime) { + maxTime = record.time; + } + } + + acquisitionInfo.setSensingStart(new Date(minTime * 1000L)); + acquisitionInfo.setSensingStop(new Date(maxTime * 1000L)); + + acquisitionInfo.setNodeType(NodeType.UNDEFINED); + + return acquisitionInfo; + } + + @Override + public String getRegEx() { + return REG_EX; + } + + @Override + public PixelLocator getPixelLocator() throws IOException { + throw new RuntimeException("not implemented"); + } + + @Override + public PixelLocator getSubScenePixelLocator(Polygon sceneGeometry) throws IOException { + throw new RuntimeException("not implemented"); + } + + @Override + public TimeLocator getTimeLocator() throws IOException { + if (timeLocator == null) { + createTimeLocator(); + } + return timeLocator; + } + + @Override + public int[] extractYearMonthDayFromFilename(String fileName) { + final int endIdx = fileName.indexOf(".txt"); + final String yearString = fileName.substring(endIdx - 7, endIdx - 3); + final String monthString = fileName.substring(endIdx - 2, endIdx); + + final int[] ymd = new int[3]; + ymd[0] = Integer.parseInt(yearString); + ymd[1] = Integer.parseInt(monthString); + ymd[2] = 1; + return ymd; + } + + @Override + public Array readRaw(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException { + final TaoRecord record = records.get(centerY); + + switch (variableName) { + case TIME: + return createResultArray(record.time, NetCDFUtils.getDefaultFillValue(DataType.INT, false), DataType.INT, interval); + + case LONGITUDE: + return createResultArray(record.longitude, NetCDFUtils.getDefaultFillValue(DataType.FLOAT, false), DataType.FLOAT, interval); + + case LATITUDE: + return createResultArray(record.latitude, NetCDFUtils.getDefaultFillValue(DataType.FLOAT, false), DataType.FLOAT, interval); + + case SSS: + return createResultArray(record.SSS, SSS_FILL, DataType.FLOAT, interval); + + case SST: + return createResultArray(record.SST, SST_FILL, DataType.FLOAT, interval); + + case AIRT: + return createResultArray(record.AIRT, AIRT_FILL, DataType.FLOAT, interval); + + case RH: + return createResultArray(record.RH, RH_FILL, DataType.FLOAT, interval); + + case WSPD: + return createResultArray(record.WSPD, WSPD_FILL, DataType.FLOAT, interval); + + case WDIR: + return createResultArray(record.WDIR, WDIR_FILL, DataType.FLOAT, interval); + + case BARO: + return createResultArray(record.BARO, BARO_FILL, DataType.FLOAT, interval); + + case RAIN: + return createResultArray(record.RAIN, RAIN_FILL, DataType.FLOAT, interval); + + case Q: + return createResultArray(record.Q, NetCDFUtils.getDefaultFillValue(DataType.INT, false), DataType.INT, interval); + + case M: + final Array resultArray = Array.factory(DataType.STRING, new int[]{1, 1}); + resultArray.setObject(0, record.M); + return resultArray; + } + + return null; + } + + @Override + public Array readScaled(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException { + return readRaw(centerX, centerY, interval, variableName); // no scaled data in this product type tb 2023-05-02 + } + + @Override + public ArrayInt.D2 readAcquisitionTime(int x, int y, Interval interval) throws IOException, InvalidRangeException { + final Array timeArray = readRaw(x, y, interval, TIME); + + return (ArrayInt.D2) timeArray; + } + + @Override + public List getVariables() throws InvalidRangeException, IOException { + final ArrayList variables = new ArrayList<>(); + + List attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "degree_east")); + attributes.add(new Attribute(CF_STANDARD_NAME, "longitude")); + variables.add(new VariableProxy(LONGITUDE, DataType.FLOAT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "degree_north")); + attributes.add(new Attribute(CF_STANDARD_NAME, "latitude")); + variables.add(new VariableProxy(LATITUDE, DataType.FLOAT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "seconds since 1970-01-01")); + attributes.add(new Attribute(CF_STANDARD_NAME, "time")); + variables.add(new VariableProxy(TIME, DataType.INT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "psu")); + attributes.add(new Attribute(CF_STANDARD_NAME, "sea_surface_salinity")); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, SSS_FILL)); + variables.add(new VariableProxy(SSS, DataType.FLOAT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "degree_Celsius")); + attributes.add(new Attribute(CF_STANDARD_NAME, "sea_surface_temperature")); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, SST_FILL)); + variables.add(new VariableProxy(SST, DataType.FLOAT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "degree_Celsius")); + attributes.add(new Attribute(CF_STANDARD_NAME, "air_temperature")); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, AIRT_FILL)); + variables.add(new VariableProxy(AIRT, DataType.FLOAT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "percent")); + attributes.add(new Attribute(CF_STANDARD_NAME, "relative_humidity")); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, RH_FILL)); + variables.add(new VariableProxy(RH, DataType.FLOAT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "m/s")); + attributes.add(new Attribute(CF_STANDARD_NAME, "wind_speed")); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, WSPD_FILL)); + variables.add(new VariableProxy(WSPD, DataType.FLOAT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "degree")); + attributes.add(new Attribute(CF_STANDARD_NAME, "wind_to_direction")); + attributes.add(new Attribute(CF_LONG_NAME, "Wind To Direction degree true in Oceanographic Convention")); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, WDIR_FILL)); + variables.add(new VariableProxy(WDIR, DataType.FLOAT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "hPa")); + attributes.add(new Attribute(CF_STANDARD_NAME, "air_pressure_at_mean_sea_level")); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, BARO_FILL)); + variables.add(new VariableProxy(BARO, DataType.FLOAT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_UNITS_NAME, "mm/hour")); + attributes.add(new Attribute(CF_STANDARD_NAME, "rainfall_rate")); + attributes.add(new Attribute(CF_FILL_VALUE_NAME, RAIN_FILL)); + variables.add(new VariableProxy(RAIN, DataType.FLOAT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_LONG_NAME, "Data Quality Codes")); + variables.add(new VariableProxy(Q, DataType.INT, attributes)); + + attributes = new ArrayList<>(); + attributes.add(new Attribute(CF_LONG_NAME, "Data Mode Codes")); + variables.add(new StringVariable(new VariableProxy(M, DataType.STRING, attributes), 8)); + + return variables; + } + + @Override + public Dimension getProductSize() throws IOException { + return new Dimension("product_size", 1, records.size()); + } + + @Override + public String getLongitudeVariableName() { + return "longitude"; + } + + @Override + public String getLatitudeVariableName() { + return "latitude"; + } + + private void createTimeLocator() { + long[] timeArray = new long[records.size()]; + + int i = 0; + for (final TaoRecord record : records) { + timeArray[i] = record.time * 1000L; + i++; + } + + timeLocator = new TimeLocator_MillisSince1970(timeArray); + } + + // @todo 2 tb/tb create generic in-itu record and move this method to it tb 2023-05-02 + private Array createResultArray(Number value, Number fillValue, DataType dataType, Interval interval) { + final int windowHeight = interval.getY(); + final int windowWidth = interval.getX(); + final Array windowArray = NetCDFUtils.create(dataType, + new int[]{windowHeight, windowWidth}, + fillValue); + + final int windowCenterX = windowWidth / 2; + final int windowCenterY = windowHeight / 2; + windowArray.setObject(windowWidth * windowCenterY + windowCenterX, value); + return windowArray; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/TaoReaderPlugin.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/TaoReaderPlugin.java new file mode 100644 index 000000000..581052a9a --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/TaoReaderPlugin.java @@ -0,0 +1,26 @@ +package com.bc.fiduceo.reader.insitu.tao; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.ReaderContext; +import com.bc.fiduceo.reader.ReaderPlugin; + +public class TaoReaderPlugin implements ReaderPlugin { + + private final static String[] SENSOR_KEYS = {"tao-sss", "pirata-sss", "rama-sss"}; + + @Override + public Reader createReader(ReaderContext readerContext) { + return new TaoReader(); + } + + @Override + public String[] getSupportedSensorKeys() { + return SENSOR_KEYS; + } + + @Override + public DataType getDataType() { + return DataType.INSITU; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/TaoRecord.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/TaoRecord.java new file mode 100644 index 000000000..4d1cbcf5d --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/TaoRecord.java @@ -0,0 +1,18 @@ +package com.bc.fiduceo.reader.insitu.tao; + +class TaoRecord { + + int time; + float longitude; + float latitude; + float SSS; + float SST; + float AIRT; + float RH; + float WSPD; + float WDIR; + float BARO; + float RAIN; + int Q; + String M; +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/AIRTRecord.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/AIRTRecord.java new file mode 100644 index 000000000..89014b250 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/AIRTRecord.java @@ -0,0 +1,9 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +class AIRTRecord { + + int date; + String AIRT; + String Q; + String M; +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/AirtProvider.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/AirtProvider.java new file mode 100644 index 000000000..80951d823 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/AirtProvider.java @@ -0,0 +1,56 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashMap; + +class AirtProvider { + + private HashMap airtMap; + + void open(File airtFile) { + airtMap = new HashMap<>(); + + if (airtFile == null) { + return; + } + + try (final FileReader fileReader = new FileReader(airtFile)) { + final BufferedReader bufferedReader = new BufferedReader(fileReader); + String line; + while ((line = bufferedReader.readLine()) != null) { + line = line.trim(); + if (!Character.isDigit(line.charAt(0))) { + continue; + } + + final AIRTRecord airtRecord = new AIRTRecord(); + final String[] tokens = TaoPreProcessor.tokenize(line); + airtRecord.date = TaoPreProcessor.toUnixEpoch(tokens[0], tokens[1]); + airtRecord.AIRT = tokens[2]; + airtRecord.Q = tokens[3]; + airtRecord.M = tokens[4]; + + airtMap.put(airtRecord.date, airtRecord); + } + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + AIRTRecord get(int date) { + AIRTRecord airtRecord = airtMap.get(date); + if (airtRecord == null) { + airtRecord = new AIRTRecord(); + airtRecord.date = date; + airtRecord.AIRT = "-9.99"; + airtRecord.Q = "9"; + airtRecord.M = "D"; + } + + return airtRecord; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/BAROProvider.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/BAROProvider.java new file mode 100644 index 000000000..90eb89800 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/BAROProvider.java @@ -0,0 +1,55 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashMap; + +class BAROProvider { + + private HashMap baroMap; + + void open(File baroFile) { + baroMap = new HashMap<>(); + + if (baroFile == null) { + return; + } + + try (final FileReader fileReader = new FileReader(baroFile)) { + final BufferedReader bufferedReader = new BufferedReader(fileReader); + String line; + while ((line = bufferedReader.readLine()) != null) { + line = line.trim(); + if (!Character.isDigit(line.charAt(0))) { + continue; + } + + final BARORecord baroRecord = new BARORecord(); + final String[] tokens = TaoPreProcessor.tokenize(line); + baroRecord.date = TaoPreProcessor.toUnixEpoch(tokens[0], tokens[1]); + baroRecord.BARO = tokens[2]; + baroRecord.Q = tokens[3]; + baroRecord.M = tokens[4]; + + baroMap.put(baroRecord.date, baroRecord); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + BARORecord get(int date) { + BARORecord baroRecord = baroMap.get(date); + if (baroRecord == null) { + baroRecord = new BARORecord(); + baroRecord.date = date; + baroRecord.BARO = "-9.9"; + baroRecord.Q = "9"; + baroRecord.M = "D"; + } + + return baroRecord; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/BARORecord.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/BARORecord.java new file mode 100644 index 000000000..bf1819818 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/BARORecord.java @@ -0,0 +1,9 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +class BARORecord { + + int date; + String BARO; + String Q; + String M; +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/Configuration.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/Configuration.java new file mode 100644 index 000000000..d7850e1b1 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/Configuration.java @@ -0,0 +1,17 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +class Configuration { + String sourceDir; + String targetDir; + String filePrefix; + String sssFileName; + String sstFileName; + String airtFileName; + String rhFileName; + String windFileName; + String baroFileName; + String rainFileName; + String posFileName; + float nominalLon; + float nominalLat; +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSProvider.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSProvider.java new file mode 100644 index 000000000..56b367b4c --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSProvider.java @@ -0,0 +1,85 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; + +class POSProvider { + + private POSRecord[] posArray; + private float nominalLon; + private float nominalLat; + + void open(File posFile, float nominalLon, float nominalLat) { + this.nominalLon = nominalLon; + this.nominalLat = nominalLat; + + if (posFile == null) { + posArray = new POSRecord[0]; + return; + } + + final ArrayList posRecords = new ArrayList<>(); + + try (final FileReader fileReader = new FileReader(posFile)) { + final BufferedReader bufferedReader = new BufferedReader(fileReader); + String line; + while ((line = bufferedReader.readLine()) != null) { + line = line.trim(); + if (!Character.isDigit(line.charAt(0))) { + continue; + } + + final POSRecord posRecord = new POSRecord(); + final String[] tokens = TaoPreProcessor.tokenize(line); + posRecord.date = TaoPreProcessor.toUnixEpoch(tokens[0], tokens[1]); + posRecord.lon = Float.parseFloat(tokens[2]); + posRecord.lat = Float.parseFloat(tokens[3]); + + posRecords.add(posRecord); + } + + posRecords.sort(Comparator.comparing(POSRecord::getDate)); + posArray = posRecords.toArray(new POSRecord[0]); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + POSRecord get(int date) { + for (int i = 0; i < posArray.length; i++) { + if (posArray[i].date < date) { + continue; + } + + if (posArray[i].date == date) { + return posArray[i]; + } + + if (i > 0) { + return interpolate(posArray[i - 1], posArray[i], date); + } else { + return posArray[0]; + } + } + + final POSRecord posRecord = new POSRecord(); + posRecord.lon = nominalLon; + posRecord.lat = nominalLat; + return posRecord; + } + + static POSRecord interpolate(POSRecord before, POSRecord after, int date) { + final POSRecord posRecord = new POSRecord(); + posRecord.date = date; + + final float factor = (float) (date - before.date) / (float) (after.date - before.date); + + posRecord.lon = before.lon + factor * (after.lon - before.lon); + posRecord.lat = before.lat + factor * (after.lat - before.lat); + return posRecord; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSRecord.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSRecord.java new file mode 100644 index 000000000..c3fe736d5 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSRecord.java @@ -0,0 +1,12 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +class POSRecord { + + int date; + float lon; + float lat; + + public int getDate() { + return date; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RAINProvider.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RAINProvider.java new file mode 100644 index 000000000..b6891e9b9 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RAINProvider.java @@ -0,0 +1,57 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashMap; + +class RAINProvider { + + private HashMap rainMap; + + void open(File rainFile) { + rainMap = new HashMap<>(); + + if (rainFile == null){ + return; + } + + try (final FileReader fileReader = new FileReader(rainFile)) { + final BufferedReader bufferedReader = new BufferedReader(fileReader); + String line; + while ((line = bufferedReader.readLine()) != null) { + line = line.trim(); + if (!Character.isDigit(line.charAt(0))) { + continue; + } + + System.out.println("line = " + line); + + final RAINRecord rainRecord = new RAINRecord(); + final String[] tokens = TaoPreProcessor.tokenize(line); + rainRecord.date = TaoPreProcessor.toUnixEpoch(tokens[0], tokens[1]); + rainRecord.RAIN = tokens[2]; + rainRecord.Q = tokens[3]; + rainRecord.M = tokens[4]; + + rainMap.put(rainRecord.date, rainRecord); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + RAINRecord get(int date) { + RAINRecord sstRecord = rainMap.get(date); + if (sstRecord == null) { + sstRecord = new RAINRecord(); + sstRecord.date = date; + sstRecord.RAIN = "-9.99"; + sstRecord.Q = "9"; + sstRecord.M = "D"; + } + + return sstRecord; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RAINRecord.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RAINRecord.java new file mode 100644 index 000000000..b481a8769 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RAINRecord.java @@ -0,0 +1,9 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +class RAINRecord { + + int date; + String RAIN; + String Q; + String M; +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RHProvider.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RHProvider.java new file mode 100644 index 000000000..5f2ff7ece --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RHProvider.java @@ -0,0 +1,55 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashMap; + +class RHProvider { + + private HashMap rhMap; + + void open(File rhFile) { + rhMap = new HashMap<>(); + + if (rhFile == null) { + return; + } + + try (final FileReader fileReader = new FileReader(rhFile)) { + final BufferedReader bufferedReader = new BufferedReader(fileReader); + String line; + while ((line = bufferedReader.readLine()) != null) { + line = line.trim(); + if (!Character.isDigit(line.charAt(0))) { + continue; + } + + final RHRecord rhRecord = new RHRecord(); + final String[] tokens = TaoPreProcessor.tokenize(line); + rhRecord.date = TaoPreProcessor.toUnixEpoch(tokens[0], tokens[1]); + rhRecord.RH = tokens[2]; + rhRecord.Q = tokens[3]; + rhRecord.M = tokens[4]; + + rhMap.put(rhRecord.date, rhRecord); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + RHRecord get(int date) { + RHRecord rhRecord = rhMap.get(date); + if (rhRecord == null) { + rhRecord = new RHRecord(); + rhRecord.date = date; + rhRecord.RH = "-9.99"; + rhRecord.Q = "9"; + rhRecord.M = "D"; + } + + return rhRecord; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RHRecord.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RHRecord.java new file mode 100644 index 000000000..84450cb1f --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/RHRecord.java @@ -0,0 +1,9 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +class RHRecord { + + int date; + String RH; + String Q; + String M; +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/SSSRecord.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/SSSRecord.java new file mode 100644 index 000000000..0dd633efb --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/SSSRecord.java @@ -0,0 +1,8 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +class SSSRecord { + int date; + String SSS; + String Q; + String M; +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/SSTProvider.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/SSTProvider.java new file mode 100644 index 000000000..caddc5d37 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/SSTProvider.java @@ -0,0 +1,55 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashMap; + +class SSTProvider { + + private HashMap sstMap; + + void open(File sstFile) { + sstMap = new HashMap<>(); + + if (sstFile == null){ + return; + } + + try (final FileReader fileReader = new FileReader(sstFile)) { + final BufferedReader bufferedReader = new BufferedReader(fileReader); + String line; + while ((line = bufferedReader.readLine()) != null) { + line = line.trim(); + if (!Character.isDigit(line.charAt(0))) { + continue; + } + + final SSTRecord sstRecord = new SSTRecord(); + final String[] tokens = TaoPreProcessor.tokenize(line); + sstRecord.date = TaoPreProcessor.toUnixEpoch(tokens[0], tokens[1]); + sstRecord.SST = tokens[2]; + sstRecord.Q = tokens[3]; + sstRecord.M = tokens[4]; + + sstMap.put(sstRecord.date, sstRecord); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + SSTRecord get(int date) { + SSTRecord sstRecord = sstMap.get(date); + if (sstRecord == null) { + sstRecord = new SSTRecord(); + sstRecord.date = date; + sstRecord.SST = "-9.999"; + sstRecord.Q = "9"; + sstRecord.M = "D"; + } + + return sstRecord; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/SSTRecord.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/SSTRecord.java new file mode 100644 index 000000000..c81864fd7 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/SSTRecord.java @@ -0,0 +1,9 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +class SSTRecord { + + int date; + String SST; + String Q; + String M; +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/TAORecord.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/TAORecord.java new file mode 100644 index 000000000..e6439f27e --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/TAORecord.java @@ -0,0 +1,21 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +class TAORecord { + int date; + String lon; + String lat; + String SSS; + String SST; + String AIRT; + String RH; + String WSPD; + String WDIR; + String BARO; + String RAIN; + String Q; + String M; + + String toLine() { + return date + " " + lon + " " + lat + " " + SSS + " " + SST + " " + AIRT + " " + RH + " " + WSPD + " " + WDIR + " " + BARO + " " + RAIN + " " + Q + " " + M; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/TaoPreProcessor.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/TaoPreProcessor.java new file mode 100644 index 000000000..514dff167 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/TaoPreProcessor.java @@ -0,0 +1,304 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import com.bc.fiduceo.util.TimeUtils; +import org.esa.snap.core.util.StringUtils; + +import java.io.*; +import java.text.DecimalFormat; +import java.util.*; + +public class TaoPreProcessor { + + public static void main(String[] args) throws IOException { + final Configuration configuration = new Configuration(); + configuration.sourceDir = "C:\\Satellite\\CIMR\\rama\\15n90e"; + configuration.targetDir = "C:\\Satellite\\CIMR\\rama_merged"; + configuration.filePrefix = "RAMA_15N90E"; + configuration.sssFileName = "sss15n90e_hr.ascii"; + configuration.sstFileName = "sst15n90e_10m.ascii"; + configuration.airtFileName = "airt15n90e_10m.ascii"; + configuration.rhFileName = "rh15n90e_10m.ascii"; + configuration.windFileName = "w15n90e_10m.ascii"; + configuration.baroFileName = "bp15n90e_hr.ascii"; + configuration.rainFileName = "rain15n90e_10m.ascii"; + configuration.posFileName = null; + configuration.nominalLon = 90.f; + configuration.nominalLat = 15.f; + + // --- read all we need --- + final File sourceDir = new File(configuration.sourceDir); + final File sssFile = new File(sourceDir, configuration.sssFileName); + + final HashMap> sssMap = parseSSSFile(sssFile); + final SSTProvider sstProvider = new SSTProvider(); + sstProvider.open(getFile(sourceDir, configuration.sstFileName)); + + final AirtProvider airtProvider = new AirtProvider(); + airtProvider.open(getFile(sourceDir, configuration.airtFileName)); + + final RHProvider rhProvider = new RHProvider(); + rhProvider.open(getFile(sourceDir, configuration.rhFileName)); + + final WINDProvider windProvider = new WINDProvider(); + windProvider.open(getFile(sourceDir, configuration.windFileName)); + + final BAROProvider baroProvider = new BAROProvider(); + baroProvider.open(getFile(sourceDir, configuration.baroFileName)); + + final RAINProvider rainProvider = new RAINProvider(); + rainProvider.open(getFile(sourceDir, configuration.rainFileName)); + + final POSProvider posProvider = new POSProvider(); + posProvider.open(getFile(sourceDir, configuration.posFileName), configuration.nominalLon, configuration.nominalLat); + + final HashMap> taoMap = new HashMap<>(); + + // --- assemble final records --- + Set deployments = sssMap.keySet(); + for (final String deployment : deployments) { + final List sssRecords = sssMap.get(deployment); + final ArrayList taoRecords = new ArrayList<>(); + + for (final SSSRecord sssRecord : sssRecords) { + final TAORecord taoRecord = new TAORecord(); + String M = ""; + String Q = ""; + + // date and place + taoRecord.date = sssRecord.date; + final POSRecord posRecord = posProvider.get(taoRecord.date); + taoRecord.lon = Float.toString(posRecord.lon); + taoRecord.lat = Float.toString(posRecord.lat); + + // salinity + taoRecord.SSS = sssRecord.SSS; + M = M.concat(sssRecord.M); + Q = Q.concat(sssRecord.Q); + + // sst data + final SSTRecord sstRecord = sstProvider.get(sssRecord.date); + taoRecord.SST = sstRecord.SST; + M = M.concat(sstRecord.M); + Q = Q.concat(sstRecord.Q); + + // airt data + final AIRTRecord airtRecord = airtProvider.get(sssRecord.date); + taoRecord.AIRT = airtRecord.AIRT; + M = M.concat(airtRecord.M); + Q = Q.concat(airtRecord.Q); + + // relative humidity + final RHRecord rhRecord = rhProvider.get(sssRecord.date); + taoRecord.RH = rhRecord.RH; + M = M.concat(rhRecord.M); + Q = Q.concat(rhRecord.Q); + + // wind + final WINDRecord windRecord = windProvider.get(sssRecord.date); + taoRecord.WSPD = windRecord.WSPD; + taoRecord.WDIR = windRecord.WDIR; + M = M.concat(windRecord.M); + Q = Q.concat(windRecord.Q); + + // barometric pressure + final BARORecord baroRecord = baroProvider.get(sssRecord.date); + taoRecord.BARO = baroRecord.BARO; + M = M.concat(baroRecord.M); + Q = Q.concat(baroRecord.Q); + + // rainfall + final RAINRecord rainRecord = rainProvider.get(sssRecord.date); + taoRecord.RAIN = rainRecord.RAIN; + M = M.concat(rainRecord.M); + Q = Q.concat(rainRecord.Q); + + taoRecord.M = M; + taoRecord.Q = Q; + + taoRecords.add(taoRecord); + } + + taoMap.put(deployment, taoRecords); + } + + // --- write files per month and deployment ---- + final DecimalFormat format = new DecimalFormat("00"); + deployments = taoMap.keySet(); + PrintWriter writer = null; + for (final String deployment : deployments) { + final Calendar utcCalendar = TimeUtils.getUTCCalendar(); + final String filePrefix = configuration.filePrefix + "_" + deployment + "_"; + + int currentYear = -9; + int currentMonth = -9; + + final List taoRecords = taoMap.get(deployment); + for (final TAORecord taoRecord : taoRecords) { + utcCalendar.setTimeInMillis(taoRecord.date * 1000L); + int year = utcCalendar.get(Calendar.YEAR); + int month = utcCalendar.get(Calendar.MONTH) + 1; + if (year != currentYear || month != currentMonth) { + currentYear = year; + currentMonth = month; + writer = switchWriter(configuration, format, writer, filePrefix, currentYear, currentMonth); + } + + writer.println(taoRecord.toLine()); + } + } + + if (writer != null) { + writer.flush(); + writer.close(); + } + } + + private static File getFile(File sourceDir, String fileName) { + if (StringUtils.isNotNullAndNotEmpty(fileName)) { + return new File(sourceDir, fileName); + } + return null; + } + + private static PrintWriter switchWriter(Configuration configuration, DecimalFormat format, PrintWriter writer, String filePrefix, int currentYear, int currentMonth) throws IOException { + if (writer != null) { + writer.flush(); + writer.close(); + } + // create new writer + final File targetDir = new File(configuration.targetDir, currentYear + File.separator + format.format(currentMonth)); + if (!targetDir.exists()) { + if (!targetDir.mkdirs()) { + throw new IOException("unable to create directory: " + targetDir.getAbsolutePath()); + } + } + + final File targetFile = new File(targetDir, filePrefix + currentYear + "-" + format.format(currentMonth) + ".txt"); + if (targetFile.exists()) { + if (!targetFile.delete()) { + throw new IOException("unable to delete file: " + targetFile.getAbsolutePath()); + } + } + if (!targetFile.createNewFile()) { + throw new IOException("unable to create file: " + targetFile.getAbsolutePath()); + } + + final PrintWriter printWriter = new PrintWriter(targetFile); + printWriter.println("# utc longitude latitude SSS SST AIRT RH WSPD WDIR BARO RAIN M Q"); + return printWriter; + } + + private static HashMap> parseSSSFile(File sssFile) { + final HashMap> sssMap = new HashMap<>(); + try (final FileReader fileReader = new FileReader(sssFile)) { + final BufferedReader bufferedReader = new BufferedReader(fileReader); + String line; + ArrayList records = new ArrayList<>(); + while ((line = bufferedReader.readLine()) != null) { + line = line.trim(); + if (line.startsWith("Platform") || line.startsWith("Parameter") || line.startsWith("YYYY") + || line.startsWith("Location") || line.startsWith("Units") || line.startsWith("Time")) { + continue; // skip these tb 2023-03-30 + } + + if (line.startsWith("Deployment")) { + // parse deployment and start new record-container + final String[] tokens = StringUtils.split(line, new char[]{' '}, true); + sssMap.put(tokens[1], records); + continue; + } + + if (line.startsWith("Depth") || line.startsWith("Height")) { + // TODO! parse depth or height and store + continue; + } + + final String[] tokens = TaoPreProcessor.tokenize(line); + final SSSRecord sssRecord = new SSSRecord(); + sssRecord.date = toUnixEpoch(tokens[0], tokens[1]); + sssRecord.SSS = tokens[2]; + if (tokens.length == 16) { + sssRecord.Q = tokens[14].substring(0, 1); + sssRecord.M = tokens[15].substring(0, 1); + } else if (tokens.length == 12) { + sssRecord.Q = tokens[10].substring(0, 1); + sssRecord.M = tokens[11].substring(0, 1); + } else if (tokens.length == 9) { + sssRecord.Q = tokens[7].substring(0, 1); + sssRecord.M = tokens[8].substring(0, 1); + } else if (tokens.length == 8) { + sssRecord.Q = tokens[6].substring(0, 1); + sssRecord.M = tokens[7].substring(0, 1); + } else if (tokens.length == 6) { + sssRecord.Q = tokens[4].substring(0, 1); + sssRecord.M = tokens[5].substring(0, 1); + } else if (tokens.length == 5) { + sssRecord.Q = tokens[3]; + sssRecord.M = tokens[4]; + } else { + throw new IOException("unknown line format:" + line); + } + + records.add(sssRecord); + } + if (sssMap.isEmpty()) { + sssMap.put("sss", records); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + + return sssMap; + } + + // Triton + // ------ + // salt YYYYMMDD HHMMSS SSS Q M depth + // airt AIRT Q M height + // baro SLP Q M height + // rain RAIN Q M height + // rh RH Q M height + // sst SST Q M depth + // wind WSPD WDIR Q M height + // lon + // lat + // 8 parameter + + // TAO + // salt YYYYMMDD HHMMSS SSS Q M depth + // airt AIRT Q M height + // baro SLP Q M height + // rain RAIN Q M height + // rh RH Q M height + // sst SST Q M depth + // wind WSPD WDIR Q M height + + static int toUnixEpoch(String yyyymmdd, String hhmmss) { + int year = Integer.parseInt(yyyymmdd.substring(0, 4)); + int month = Integer.parseInt(yyyymmdd.substring(4, 6)); + int day = Integer.parseInt(yyyymmdd.substring(6, 8)); + + int hour = Integer.parseInt(hhmmss.substring(0, 2)); + int minute = Integer.parseInt(hhmmss.substring(2, 4)); + int second = 0; + if (hhmmss.length() > 4) { + second = Integer.parseInt(hhmmss.substring(4, 6)); + } + + final Calendar calendar = TimeUtils.getUTCCalendar(); + calendar.set(Calendar.YEAR, year); + calendar.set(Calendar.MONTH, month - 1); + calendar.set(Calendar.DAY_OF_MONTH, day); + calendar.set(Calendar.HOUR_OF_DAY, hour); + calendar.set(Calendar.MINUTE, minute); + calendar.set(Calendar.SECOND, second); + + long timeInMillis = calendar.getTime().getTime(); + return (int) (timeInMillis / 1000); + } + + static String[] tokenize(String line) { + line = line.replaceAll(" +", " "); // some fields are separated by two or more blanks (sigh) tb 2023-04-12 + return StringUtils.split(line, new char[]{' '}, true); + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/WINDProvider.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/WINDProvider.java new file mode 100644 index 000000000..330682eae --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/WINDProvider.java @@ -0,0 +1,62 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashMap; + +class WINDProvider { + + private HashMap windMap; + + void open(File windFile) { + windMap = new HashMap<>(); + + if (windFile == null) { + return; + } + + try (final FileReader fileReader = new FileReader(windFile)) { + final BufferedReader bufferedReader = new BufferedReader(fileReader); + String line; + while ((line = bufferedReader.readLine()) != null) { + line = line.trim(); + if (!Character.isDigit(line.charAt(0))) { + continue; + } + + final WINDRecord windRecord = new WINDRecord(); + final String[] tokens = TaoPreProcessor.tokenize(line); + windRecord.date = TaoPreProcessor.toUnixEpoch(tokens[0], tokens[1]); + windRecord.WSPD = tokens[4]; + windRecord.WDIR = tokens[5]; + if (tokens[6].length() > 2) { + windRecord.Q = tokens[6].substring(2, 4); + windRecord.M = tokens[7].substring(2, 4); + } else { + windRecord.Q = tokens[6]; + windRecord.M = tokens[7]; + } + + windMap.put(windRecord.date, windRecord); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + WINDRecord get(int date) { + WINDRecord windRecord = windMap.get(date); + if (windRecord == null) { + windRecord = new WINDRecord(); + windRecord.date = date; + windRecord.WSPD = "-99.9"; + windRecord.WDIR = "-99.9"; + windRecord.Q = "99"; + windRecord.M = "DD"; + } + + return windRecord; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/WINDRecord.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/WINDRecord.java new file mode 100644 index 000000000..0135248e3 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/insitu/tao/preproc/WINDRecord.java @@ -0,0 +1,9 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +class WINDRecord { + int date; + String WSPD; + String WDIR; + String Q; + String M; +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/slstr/SlstrReaderConfig.java b/core/src/main/java/com/bc/fiduceo/reader/slstr/SlstrReaderConfig.java index 13410c352..e648121a3 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/slstr/SlstrReaderConfig.java +++ b/core/src/main/java/com/bc/fiduceo/reader/slstr/SlstrReaderConfig.java @@ -3,10 +3,10 @@ import com.bc.fiduceo.core.SystemConfig; import com.bc.fiduceo.log.FiduceoLogger; import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.JDOMException; -import org.jdom.input.SAXBuilder; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.JDOMException; +import org.jdom2.input.SAXBuilder; import java.io.File; import java.io.FileInputStream; diff --git a/core/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReader.java b/core/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReader.java index 4919c7512..bef553aa4 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReader.java +++ b/core/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReader.java @@ -20,8 +20,8 @@ import com.bc.fiduceo.store.ZipStore; import com.bc.fiduceo.util.NetCDFUtils; import com.bc.fiduceo.util.TimeUtils; -import org.esa.s3tbx.dataio.s3.Manifest; -import org.esa.s3tbx.dataio.s3.XfduManifest; +import eu.esa.opt.dataio.s3.Manifest; +import eu.esa.opt.dataio.s3.XfduManifest; import org.esa.snap.core.dataio.geocoding.*; import org.esa.snap.core.dataio.geocoding.forward.PixelForward; import org.esa.snap.core.dataio.geocoding.inverse.PixelQuadTreeInverse; diff --git a/core/src/main/java/com/bc/fiduceo/reader/smap/SmapPixelLocator.java b/core/src/main/java/com/bc/fiduceo/reader/smap/SmapPixelLocator.java new file mode 100644 index 000000000..8b2794390 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/smap/SmapPixelLocator.java @@ -0,0 +1,205 @@ +package com.bc.fiduceo.reader.smap; + +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.location.PixelLocator; +import com.bc.fiduceo.math.SphericalDistance; +import com.bc.fiduceo.reader.RawDataReader; +import com.bc.fiduceo.util.NetCDFUtils; +import ucar.ma2.Array; +import ucar.ma2.Index; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Variable; + +import java.awt.geom.Point2D; +import java.io.IOException; +import java.util.ArrayList; + +class SmapPixelLocator implements PixelLocator { + + private static final String YDIM_GRID = "ydim_grid"; + private static final String XDIM_GRID = "xdim_grid"; + private static final String LOOK = "look"; + private static final int CELL_COUNT_360_DEGREE = 360 * 4; + private static final String CF_FillValue = NetCDFUtils.CF_FILL_VALUE_NAME; + + private final Array lons; + private final Array lats; + private final int fullHeight; + private final int fullWidth; + private final float fillValue; + private final int xIdx; + private final int yIdx; + private final double degree360ColIdx; + + SmapPixelLocator(Variable cellonVar, Variable cellatVar, int lookVal) throws IOException { + + yIdx = cellonVar.findDimensionIndex(YDIM_GRID); + fullHeight = cellonVar.getDimension(yIdx).getLength(); + xIdx = cellonVar.findDimensionIndex(XDIM_GRID); + fullWidth = cellonVar.getDimension(xIdx).getLength(); + + fillValue = cellonVar.findAttribute(CF_FillValue).getNumericValue().floatValue(); + + final int lookIdx = cellonVar.findDimensionIndex(LOOK); + final int[] shape = cellonVar.getShape(); + shape[lookIdx] = 1; + final int[] origin = new int[shape.length]; + origin[lookIdx] = lookVal; + + try { + lons = cellonVar.read(origin, shape).reduce(); + lats = cellatVar.read(origin, shape).reduce(); + degree360ColIdx = find360degreeIndex() + 0.5; + } catch (InvalidRangeException e) { + throw new IOException(e); + } + + } + + @Override + public Point2D getGeoLocation(double x, double y, Point2D g) { + if (x<0 || x>=fullWidth|| y<0||y>=fullHeight) { + return null; + } + final Index index = lons.getIndex(); + index.set((int) Math.floor(y), (int) Math.floor(x)); + final float lon = lons.getFloat(index); + final float lat = lats.getFloat(index); + if (lon != fillValue && lat != fillValue) { + if (g != null) { + g.setLocation(lon, lat); + return g; + } + return new Point2D.Float(lon, lat); + } + return null; + } + + @Override + public Point2D[] getPixelLocation(double lon, double lat) { + if (lat > 90 || lat < -90) { + return new Point2D[0]; + } + lon = ensureLonRange360(lon); + double lonIDX = degree360ColIdx - 1 - Math.floor(lon * 4); + while (lonIDX < 0) { + lonIDX += CELL_COUNT_360_DEGREE; + } + final double latIDX = Math.floor((lat + 90) * 4) + 0.5; + + final ArrayList validPixels = new ArrayList<>(); + addClosestValidPixel(lonIDX, latIDX, lon, lat, validPixels); + + final double secondLonIDX = lonIDX + CELL_COUNT_360_DEGREE; + if (secondLonIDX < fullWidth) { + addClosestValidPixel(secondLonIDX, latIDX, lon, lat, validPixels); + } + return validPixels.toArray(new Point2D[0]); + } + + private void addClosestValidPixel(double lonIDX, double latIDX, double lon, double lat, ArrayList validPixels) { + try { + final Dimension productSize = new Dimension("size", fullWidth, fullHeight); + final Interval readSize = new Interval(3, 3); + final int centerX = (int) lonIDX; + final int centerY = (int) latIDX; + final float[] lon3x3 = (float[]) RawDataReader.read(centerX, centerY, readSize, fillValue, lons, productSize).copyTo1DJavaArray(); + final float[] lat3x3 = (float[]) RawDataReader.read(centerX, centerY, readSize, fillValue, lats, productSize).copyTo1DJavaArray(); + double min = Double.MAX_VALUE; + int minIdx = -1; + final SphericalDistance sd = new SphericalDistance(lon, lat); + for (int i = 0; i < lon3x3.length; i++) { + float lon2 = lon3x3[i]; + float lat2 = lat3x3[i]; + if (lon2 == fillValue || lat2 == fillValue) { + continue; + } + final double dist = sd.distance(lon2, lat2); + if (dist < min) { + min = dist; + minIdx = i; + } + } + if (minIdx > -1) { + final int corrLonIDX = minIdx % 3 - 1; + final int corrLatIDX = minIdx / 3 - 1; + validPixels.add(new Point2D.Double(lonIDX + corrLonIDX, latIDX + corrLatIDX)); + } + } catch (IOException e) { + // This should never happen, because the data is already loaded. + throw new RuntimeException(e); + } + } + + private static double ensureLonRange360(double lon) { + while (lon < 0) { + lon += 360; + } + while (lon > 360) { + lon -= 360; + } + if (lon == 360.0) { + lon = 0.0; + } + return lon; + } + + private int find360degreeIndex() throws InvalidRangeException { + final int[] verticalShape = new int[]{fullHeight, 1}; + final int[] origin = new int[2]; + SectionInfo mostValuesSection = new SectionInfo(0, 0, -1); + int mostValuesIdx = 0; + for (int x = 0; x < fullWidth; x++) { + origin[xIdx] = x; + final Array sectArr = lons.section(origin, verticalShape); + final SectionInfo si = getSectionInfo((float[]) sectArr.copyTo1DJavaArray()); +// if (Double.MAX_VALUE==si.min) { +// System.out.printf("%4d ---,--- ---,--- ---,--- \n", x, si.min, si.max + ((si.max - si.min) / 2), si.max); +// } else { +// System.out.printf("%4d %7.3f %7.3f %7.3f \n", x, si.min, si.min + ((si.max - si.min) / 2), si.max); +// } + if (si.validCount > mostValuesSection.validCount) { + mostValuesSection = si; + mostValuesIdx = x; + } + } + final double min = mostValuesSection.min; + final int quarterDegree0To360CellIndex = (int) Math.floor(min * 4); + final int offsetTo360DegreeCell = CELL_COUNT_360_DEGREE - 1 - quarterDegree0To360CellIndex; + return mostValuesIdx - offsetTo360DegreeCell; + } + + private SectionInfo getSectionInfo(float[] slice) { + double max = -Double.MAX_VALUE; + double min = Double.MAX_VALUE; + int count = 0; + for (float val : slice) { + if (val == fillValue) { + continue; + } + count++; + if (val > max) { + max = val; + } + if (val < min) { + min = val; + } + } + return new SectionInfo(min, max, count); + } + + static class SectionInfo { + final double min; + final double max; + final int validCount; + + private int x; + + SectionInfo(double min, double max, int validCount) { + this.min = min; + this.max = max; + this.validCount = validCount; + } + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/smap/SmapReader.java b/core/src/main/java/com/bc/fiduceo/reader/smap/SmapReader.java new file mode 100644 index 000000000..e3f2e9b33 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/smap/SmapReader.java @@ -0,0 +1,483 @@ +package com.bc.fiduceo.reader.smap; + +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.core.NodeType; +import com.bc.fiduceo.geometry.GeometryFactory; +import com.bc.fiduceo.geometry.GeometryUtil; +import com.bc.fiduceo.geometry.L3TimeAxis; +import com.bc.fiduceo.geometry.MultiLineString; +import com.bc.fiduceo.geometry.Polygon; +import com.bc.fiduceo.geometry.TimeAxis; +import com.bc.fiduceo.location.PixelLocator; +import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.RawDataReader; +import com.bc.fiduceo.reader.ReaderContext; +import com.bc.fiduceo.reader.ReaderUtils; +import com.bc.fiduceo.reader.netcdf.NetCDFReader; +import com.bc.fiduceo.reader.time.TimeLocator; +import com.bc.fiduceo.reader.time.TimeLocator_SecondsSince2000; +import com.bc.fiduceo.util.NetCDFUtils; +import com.bc.fiduceo.util.TimeUtils; +import com.bc.fiduceo.util.VariableProxy; +import org.apache.commons.collections.MapUtils; +import org.esa.snap.dataio.netcdf.PartialDataCopier; +import ucar.ma2.Array; +import ucar.ma2.ArrayInt; +import ucar.ma2.InvalidRangeException; +import ucar.ma2.MAMath; +import ucar.nc2.Variable; + +import java.awt.geom.Rectangle2D; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +class SmapReader extends NetCDFReader { + + /** + * For detailed information about the meaning of "look" please see page 56 in the document: + * https://data.remss.com/smap/SSS/V05.0/documents/SMAP_NASA_RSS_Salinity_Release_V5.0.pdf + */ + public enum LOOK { + // Since index positions are zero based, 0 means "look 1". And "look 1" means "for look". + FOR(0), + // Since index positions are zero based, 1 means "look 2". And "look 2" means "aft look". + AFT(1); + + private final int index; + + LOOK(int val) { + this.index = val; + } + + public int getIndex() { + return index; + } + } + + private static final String DIM_NAME_X = "xdim_grid"; + private static final String DIM_NAME_Y = "ydim_grid"; + private static final String CF_FillValue = NetCDFUtils.CF_FILL_VALUE_NAME; + + // DATA FORMAT SPECIFICATION + // at: https://data.remss.com/smap/SSS/V05.0/documents/SMAP_NASA_RSS_Salinity_Release_V5.0.pdf + // Page 55 / Chapter: 12 + // see chapter 12.1.1 + private static final String REG_EX = "RSS_SMAP_SSS_L2C_r\\d{5}_\\d{8}T\\d{6}_\\d{7}_FNL_V05\\.0\\.nc"; + + // see page 56 find: "look = 2 (1 = for look, 2= aft look)" + // zero based indexes are: 0 = for look, 1= aft look" + private static final String[] LOOKS = {"for", "aft"}; + + // see table at page 44 + private static final String[] UNCERTAINTIES = + {"ws-ran", "nedt-v", "nedt-h", "sst", "wdir", "ref-gal", "lnd-ctn", "si-ctn", "ws-sys"}; + + // see iceflag_components at page 56 + private static final String[] ICEFLAGS = {"ice1", "ice2", "ice3"}; + + // see pages 56, 58 and 59 + private static final String[][] POLARIZATIONS = { + {"V", "H"}, // see polarization_2 at page 58 + {"I", "Q", "S3"}, // see polarization_3 at page 59 + {"V", "H", "S3", "S4"} // see polarization_4 at pages 58 and 59 + }; + + private static final Map VAR_DIM_EXTENSIONS = MapUtils.putAll(new HashMap(), new Object[]{ + "look", LOOKS, + "uncertainty_components", UNCERTAINTIES, + "iceflag_components", ICEFLAGS, + "polarization_2", POLARIZATIONS[0], + "polarization_3", POLARIZATIONS[1], + "polarization_4", POLARIZATIONS[2], + }); + + private final Map variableInfos = new LinkedHashMap<>(); + + private final GeometryFactory geometryFactory; + final int lookValue; + private final String lookExtName; + + private PixelLocator pixelLocator; + private TimeLocator timeLocator; + private int pWidth; + private int pHeight; + private ArrayList variables; + + /** + * Package local constructor of the SMAP data Reader. + * Should not be instantiated directly but via the plugins + * {@link SmapReaderPluginForLook#createReader(ReaderContext)} and + * {@link SmapReaderPluginAftLook#createReader(ReaderContext)}. + * For more detailed information about the meaning of "for look" and "aft look" please see page 56 in the document: + * https://data.remss.com/smap/SSS/V05.0/documents/SMAP_NASA_RSS_Salinity_Release_V5.0.pdf + * + * @param readerContext + * @param look + */ + SmapReader(ReaderContext readerContext, LOOK look) { + this.geometryFactory = readerContext.getGeometryFactory(); + + this.lookValue = look.getIndex(); + this.lookExtName = LOOKS[lookValue]; + } + + @Override + public void open(File file) throws IOException { + super.open(file); + pWidth = netcdfFile.findDimension(DIM_NAME_X).getLength(); + pHeight = netcdfFile.findDimension(DIM_NAME_Y).getLength(); + variables = initVariables(); + } + + @Override + public void close() throws IOException { + super.close(); + pixelLocator = null; + timeLocator = null; + variableInfos.clear(); + variables.clear(); + } + + @Override + public AcquisitionInfo read() throws IOException { + final AcquisitionInfo info = new AcquisitionInfo(); + + final double[] geoMinMax = extractGeoMinMax(); + final Polygon polygon = GeometryUtil.createPolygonFromMinMax(geoMinMax, geometryFactory); + info.setBoundingGeometry(polygon); + + final String startTimeString = NetCDFUtils.getGlobalAttributeString("time_coverage_start", netcdfFile); + info.setSensingStart(TimeUtils.parse(startTimeString, "yyyy-MM-dd'T'HH:mm:ss'Z'")); + + final String endTimeString = NetCDFUtils.getGlobalAttributeString("time_coverage_end", netcdfFile); + info.setSensingStop(TimeUtils.parse(endTimeString, "yyyy-MM-dd'T'HH:mm:ss'Z'")); + + final MultiLineString multiLineString = GeometryUtil.createMultiLineStringFromMinMax(geoMinMax, geometryFactory); + final TimeAxis timeAxis = new L3TimeAxis(info.getSensingStart(), info.getSensingStop(), multiLineString); + info.setTimeAxes(new TimeAxis[]{timeAxis}); + + info.setNodeType(NodeType.UNDEFINED); + + return info; + } + + @Override + public String getRegEx() { + return REG_EX; + } + + @Override + public PixelLocator getPixelLocator() throws IOException { + if (pixelLocator == null) { + final Variable cellonVar = netcdfFile.findVariable("cellon"); + final Variable cellatVar = netcdfFile.findVariable("cellat"); + + pixelLocator = new SmapPixelLocator(cellonVar, cellatVar, lookValue); + } + + return pixelLocator; + } + + @Override + public PixelLocator getSubScenePixelLocator(Polygon sceneGeometry) throws IOException { + return getPixelLocator(); + } + + @Override + public TimeLocator getTimeLocator() throws IOException { + if (timeLocator == null) { + final Array timeArray = arrayCache.get("time"); + final Number fillValue = getFillValue("time", timeArray); + final int[] origin = {0, 0, lookValue}; // select "look" layer + final int[] shape = timeArray.getShape(); + shape[2] = 1; // one "look" layer + + try { + final Array timeArraySection = timeArray.section(origin, shape).reduce(); + timeLocator = new TimeLocator_SecondsSince2000(timeArraySection, fillValue.doubleValue()); + } catch (InvalidRangeException e) { + throw new IOException(e.getMessage()); + } + } + return timeLocator; + } + + @Override + public int[] extractYearMonthDayFromFilename(String fileName) { + final int startIndex = 24; + final int year = Integer.parseInt(fileName.substring(startIndex, startIndex + 4)); + final int month = Integer.parseInt(fileName.substring(startIndex + 4, startIndex + 6)); + final int day = Integer.parseInt(fileName.substring(startIndex + 6, startIndex + 8)); + + return new int[]{year, month, day}; + } + + @Override + public Array readRaw(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException { + final VariableInfo variableInfo = variableInfos.get(variableName); + final int[] offset = variableInfo.offset; + final int[] windowShape = new int[offset.length]; + Arrays.fill(windowShape, 1); + + final Variable variable = netcdfFile.findVariable(variableInfo.ncVarName); + + final int winWith = interval.getX(); + final int winHeight = interval.getY(); + final int offsetX = centerX - winWith / 2; + final int offsetY = centerY - winHeight / 2; + final boolean isWindowInside = RawDataReader.isWindowInside(offsetX, offsetY, winWith, winHeight, pWidth, pHeight); + if (isWindowInside) { + offset[variableInfo.xDimIndex] = offsetX; + offset[variableInfo.yDimIndex] = offsetY; + windowShape[variableInfo.xDimIndex] = winWith; + windowShape[variableInfo.yDimIndex] = winHeight; + return variable.read(offset, windowShape).reduce(); + } else { + final Rectangle2D insideWindow = RawDataReader.getInsideWindow(offsetX, offsetY, winWith, winHeight, pWidth, pHeight); + offset[variableInfo.xDimIndex] = (int) insideWindow.getX(); + offset[variableInfo.yDimIndex] = (int) insideWindow.getY(); + windowShape[variableInfo.xDimIndex] = (int) insideWindow.getWidth(); + windowShape[variableInfo.yDimIndex] = (int) insideWindow.getHeight(); + final Array insideRead = variable.read(offset, windowShape).reduce(); + final Array a = NetCDFUtils.create(insideRead.getDataType(), new int[]{winHeight, winWith}, variableInfo.fillValue); + final int targetOffsY = Math.min(offsetY, 0); + final int targetOffsX = Math.min(offsetX, 0); + PartialDataCopier.copy(new int[]{targetOffsY, targetOffsX}, insideRead, a); + return a; + } + } + + private Number getFillValue(String variableName, Array array) throws IOException { + Number fillValue = arrayCache.getNumberAttributeValue(NetCDFUtils.CF_FILL_VALUE_NAME, variableName); + if (fillValue == null) { + fillValue = NetCDFUtils.getDefaultFillValue(array); + } + return fillValue; + } + + // @Override + public Array readScaled(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException { + // Since SMAP variables apparently have neither an add_offset nor a scale_factor != 1.0, the results are the same as readRaw. + return readRaw(centerX, centerY, interval, variableName); // everything is already scaled se 2023-04-12 + } + + @Override + public ArrayInt.D2 readAcquisitionTime(int x, int y, Interval interval) throws IOException, InvalidRangeException { + final Dimension productSize = getProductSize(); + final TimeLocator timeLocator = getTimeLocator(); + return ReaderUtils.readAcquisitionTimeFromTimeLocator(x, y, interval, productSize, timeLocator); + } + + @Override + public List getVariables() throws InvalidRangeException, IOException { + return Collections.unmodifiableList(variables); + } + + private ArrayList initVariables() { + final List variablesInFile = netcdfFile.getVariables(); + final ArrayList exportVariables = new ArrayList<>(); + + for (final Variable variable : variablesInFile) { + final int rank = variable.getRank(); + final String ncVarName = variable.getShortName(); + final int xDimIdx = variable.findDimensionIndex(DIM_NAME_X); + final int yDimIdx = variable.findDimensionIndex(DIM_NAME_Y); + final Number fillValue = variable.findAttribute(CF_FillValue).getNumericValue(); + if (rank == 2) { + exportVariables.add(variable); + variableInfos.put(ncVarName, new VariableInfo(ncVarName, xDimIdx, yDimIdx, new int[rank], fillValue)); + } else { + final List dimensions = variable.getDimensions(); + final List dimsToIterate = new ArrayList<>(); + final List dimIndex = new ArrayList<>(); + final List dimNamesToIterate = new ArrayList<>(); + for (ucar.nc2.Dimension dimension : dimensions) { + final String dimName = dimension.getShortName(); + if (VAR_DIM_EXTENSIONS.containsKey(dimName)) { + if ("look".equals(dimName)) { + dimsToIterate.add(new String[]{lookExtName}); + } else { + dimsToIterate.add(VAR_DIM_EXTENSIONS.get(dimName)); + } + dimIndex.add(variable.findDimensionIndex(dimName)); + dimNamesToIterate.add(dimName); + } + } + final int[] dimSliceOffset = new int[dimsToIterate.size()]; + final int[] dimSliceOffsetMax = new int[dimsToIterate.size()]; + for (int i = 0; i < dimSliceOffsetMax.length; i++) { + final String dimName = dimNamesToIterate.get(i); + if ("look".equals(dimName)) { + dimSliceOffsetMax[i] = lookValue + 1; + dimSliceOffset[i] = lookValue; + } else { + dimSliceOffsetMax[i] = dimsToIterate.get(i).length; + } + } + do { + String varName2D = ncVarName; + for (int i = 0; i < dimsToIterate.size(); i++) { + final String dimName = dimNamesToIterate.get(i); + if ("look".equals(dimName)) { + varName2D += "_" + lookExtName; + } else { + String[] varExt = dimsToIterate.get(i); + varName2D += "_" + varExt[dimSliceOffset[i]]; + } + } + exportVariables.add(new VariableProxy(varName2D, variable.getDataType(), variable.attributes().getAttributes())); + final int[] offset = new int[rank]; + for (int i = 0; i < dimSliceOffset.length; i++) { + final int dimIdx = dimIndex.get(i); + offset[dimIdx] = dimSliceOffset[i]; + } + variableInfos.put(varName2D, new VariableInfo(ncVarName, xDimIdx, yDimIdx, offset, fillValue)); + for (int i = dimSliceOffset.length - 1; i >= 0; i--) { // !!! reverse order + dimSliceOffset[i] = dimSliceOffset[i] + 1; + if (i > 0 && dimSliceOffset[i] == dimSliceOffsetMax[i]) { + if ("look".equals(dimNamesToIterate.get(i))) { + dimSliceOffset[i] = lookValue; + } else { + dimSliceOffset[i] = 0; + } + } else { + break; // leave the loop and don't increase the lower index position + } + } + } while (dimSliceOffset[0] < dimSliceOffsetMax[0]); + } + } + return exportVariables; + } + + @Override + public Dimension getProductSize() throws IOException { + return new Dimension("size", pWidth, pHeight); + } + + @Override + public String getLongitudeVariableName() { + return "cellon"; + } + + @Override + public String getLatitudeVariableName() { + return "cellat"; + } + + private double[] extractGeoMinMax() throws IOException { + final int dimIdxY = 0; + final int dimIdxX = 1; + final int dimIdxL = 2; // look + + final Variable lonVar = netcdfFile.findVariable(getLongitudeVariableName()); + final float fillValue = lonVar.findAttribute(CF_FillValue).getNumericValue().floatValue(); + + final Array lonArr = arrayCache.get(getLongitudeVariableName()); + final int[] verticalSection = lonArr.getShape(); + final int height = verticalSection[dimIdxY]; + final int width = verticalSection[dimIdxX]; + + verticalSection[dimIdxX] = 1; + verticalSection[dimIdxL] = 1; + final int[] lonOrigin = new int[verticalSection.length]; + lonOrigin[dimIdxL] = lookValue; + + final Array latArr = arrayCache.get(getLatitudeVariableName()); + final int[] horizontalSection = latArr.getShape(); + + horizontalSection[dimIdxY] = 1; + horizontalSection[dimIdxL] = 1; + final int[] latOrigin = new int[horizontalSection.length]; + latOrigin[dimIdxL] = lookValue; + + double lonMin = Double.MAX_VALUE; + double lonMax = -Double.MAX_VALUE; + double latMin = Double.MAX_VALUE; + double latMax = -Double.MAX_VALUE; + try { + for (int i = 0; i < width; i++) { + lonOrigin[dimIdxX] = i; + final Array section = lonArr.section(lonOrigin, verticalSection); + final MAMath.MinMax minMax = MAMath.getMinMaxSkipMissingData(section, fillValue); + if (minMax.min == Double.MAX_VALUE) { + continue; + } + while (minMax.min > 180) { + minMax.min -= 360; + } + if (minMax.min < lonMin) { + lonMin = minMax.min; + } + + while (minMax.max > 180) { + minMax.max -= 360; + } + if (minMax.max > lonMax) { + lonMax = minMax.max; + } + } + // Find latitude maximum --> the product is mirrored vertically --> positive values are at the lower end. + for (int i = height - 1; i >= 0; i--) { // bottom up + latOrigin[dimIdxY] = i; + final Array section = latArr.section(latOrigin, horizontalSection); + final MAMath.MinMax minMax = MAMath.getMinMaxSkipMissingData(section, fillValue); + if (minMax.max == latMax) { + continue; + } + latMax = minMax.max; + break; + } + // Find latitude minimum --> the product is mirrored vertically --> negative values are at the upper end. + for (int i = 0; i < height; i++) { // top down + latOrigin[dimIdxY] = i; + final Array section = latArr.section(latOrigin, horizontalSection); + final MAMath.MinMax minMax = MAMath.getMinMaxSkipMissingData(section, fillValue); + if (minMax.min == latMin) { + continue; + } + latMin = minMax.min; + break; + } + } catch (InvalidRangeException e) { + throw new IOException(e); + } + + return new double[]{lonMin, lonMax, latMin, latMax}; + } + + private class VariableInfo { + final String ncVarName; + final int xDimIndex; + final int yDimIndex; + final int[] offset; + final Number fillValue; + + private VariableInfo(String ncVarName, int xDimIndex, int yDimIndex, int[] offset, Number fillValue) { + this.ncVarName = ncVarName; + this.xDimIndex = xDimIndex; + this.yDimIndex = yDimIndex; + this.offset = offset; + this.fillValue = fillValue; + } + + @Override + public String toString() { + return "VariableInfo{" + + "ncVarName='" + ncVarName + '\'' + + ", xDimIndex=" + xDimIndex + + ", yDimIndex=" + yDimIndex + + ", offset=" + Arrays.toString(offset) + + ", fillValue=" + fillValue + + '}'; + } + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/smap/SmapReaderPluginAftLook.java b/core/src/main/java/com/bc/fiduceo/reader/smap/SmapReaderPluginAftLook.java new file mode 100644 index 000000000..5cb5560e5 --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/smap/SmapReaderPluginAftLook.java @@ -0,0 +1,32 @@ +package com.bc.fiduceo.reader.smap; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.ReaderContext; +import com.bc.fiduceo.reader.ReaderPlugin; +import com.bc.fiduceo.reader.smap.SmapReader.LOOK; + +/** + * Aft look Reader Plugin for SMAP Salinity data products. + * For more detailed information about the meaning of "aft look" please see page 56 in the document: + * https://data.remss.com/smap/SSS/V05.0/documents/SMAP_NASA_RSS_Salinity_Release_V5.0.pdf + */ +public class SmapReaderPluginAftLook implements ReaderPlugin { + + private static final String[] SUPPORTED_KEYS = {"smap-sss-aft"}; + + @Override + public Reader createReader(ReaderContext readerContext) { + return new SmapReader(readerContext, LOOK.AFT); + } + + @Override + public String[] getSupportedSensorKeys() { + return SUPPORTED_KEYS; + } + + @Override + public DataType getDataType() { + return DataType.POLAR_ORBITING_SATELLITE; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/smap/SmapReaderPluginForLook.java b/core/src/main/java/com/bc/fiduceo/reader/smap/SmapReaderPluginForLook.java new file mode 100644 index 000000000..67f15294c --- /dev/null +++ b/core/src/main/java/com/bc/fiduceo/reader/smap/SmapReaderPluginForLook.java @@ -0,0 +1,32 @@ +package com.bc.fiduceo.reader.smap; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.ReaderContext; +import com.bc.fiduceo.reader.ReaderPlugin; +import com.bc.fiduceo.reader.smap.SmapReader.LOOK; + +/** + * For look Reader Plugin for SMAP Salinity data products. + * For more detailed information about the meaning of "for look" please see page 56 in the document: + * https://data.remss.com/smap/SSS/V05.0/documents/SMAP_NASA_RSS_Salinity_Release_V5.0.pdf + */ +public class SmapReaderPluginForLook implements ReaderPlugin { + + private static final String[] SUPPORTED_KEYS = {"smap-sss-for"}; + + @Override + public Reader createReader(ReaderContext readerContext) { + return new SmapReader(readerContext, LOOK.FOR); + } + + @Override + public String[] getSupportedSensorKeys() { + return SUPPORTED_KEYS; + } + + @Override + public DataType getDataType() { + return DataType.POLAR_ORBITING_SATELLITE; + } +} diff --git a/core/src/main/java/com/bc/fiduceo/reader/avhrr_frac/AVHRR_FRAC_TimeLocator.java b/core/src/main/java/com/bc/fiduceo/reader/time/TimeLocator_StartStopDate.java similarity index 63% rename from core/src/main/java/com/bc/fiduceo/reader/avhrr_frac/AVHRR_FRAC_TimeLocator.java rename to core/src/main/java/com/bc/fiduceo/reader/time/TimeLocator_StartStopDate.java index e0f368c27..c6fa29c6b 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/avhrr_frac/AVHRR_FRAC_TimeLocator.java +++ b/core/src/main/java/com/bc/fiduceo/reader/time/TimeLocator_StartStopDate.java @@ -1,15 +1,13 @@ -package com.bc.fiduceo.reader.avhrr_frac; - -import com.bc.fiduceo.reader.time.TimeLocator; +package com.bc.fiduceo.reader.time; import java.util.Date; -class AVHRR_FRAC_TimeLocator implements TimeLocator { +public class TimeLocator_StartStopDate implements TimeLocator { private final long startTime; private final double increment; - AVHRR_FRAC_TimeLocator(Date startTime, Date stopTime, int numLines) { + public TimeLocator_StartStopDate(Date startTime, Date stopTime, int numLines) { this.startTime = startTime.getTime(); double timeDelta = (stopTime.getTime() - this.startTime); increment = timeDelta / (numLines - 1); diff --git a/core/src/main/java/com/bc/fiduceo/reader/windsat/WindsatReader.java b/core/src/main/java/com/bc/fiduceo/reader/windsat/WindsatReader.java index 1b4c2a8f6..1bce3476f 100644 --- a/core/src/main/java/com/bc/fiduceo/reader/windsat/WindsatReader.java +++ b/core/src/main/java/com/bc/fiduceo/reader/windsat/WindsatReader.java @@ -15,9 +15,12 @@ import com.bc.fiduceo.util.NetCDFUtils; import com.bc.fiduceo.util.TimeUtils; import com.bc.fiduceo.util.VariableProxy; +import org.esa.snap.dataio.netcdf.PartialDataCopier; import ucar.ma2.*; +import ucar.nc2.AttributeContainer; import ucar.nc2.Variable; +import java.awt.geom.Rectangle2D; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -36,6 +39,10 @@ class WindsatReader extends NetCDFReader { private PixelLocator pixelLocator; private TimeLocator timeLocator; + private Dimension productSize; + private int productWidth; + private int productHeight; + WindsatReader(ReaderContext readerContext) { this.geometryFactory = readerContext.getGeometryFactory(); variables2D = new ArrayList<>(); @@ -74,6 +81,7 @@ static int getIndex(String value, String[] strings) { @Override public void open(File file) throws IOException { super.open(file); + initProductSize(); } @Override @@ -134,15 +142,15 @@ public PixelLocator getSubScenePixelLocator(Polygon sceneGeometry) throws IOExce @Override public TimeLocator getTimeLocator() throws IOException { if (timeLocator == null) { - final Array timeArray = arrayCache.get("time"); - final Number fillValue = arrayCache.getNumberAttributeValue("_FillValue", "time"); + final Variable variable = netcdfFile.findVariable("time"); + final Number fillValue = NetCDFUtils.getFillValue(variable); final int[] origin = {0, 0, 0, 2}; // select layer for "fore" and 18 GHz tb 2022-11-28 - final int[] shape = timeArray.getShape(); + final int[] shape = variable.getShape(); shape[2] = 1; // one view layer tb 2022-11-28 shape[3] = 1; // one frequency layer tb 2022-11-28 try { - final Array timeArraySection = timeArray.section(origin, shape).reduce(); + final Array timeArraySection = variable.read(origin, shape).reduce(); timeLocator = new TimeLocator_SecondsSince2000(timeArraySection, fillValue.doubleValue()); } catch (InvalidRangeException e) { throw new IOException(e.getMessage()); @@ -166,7 +174,7 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl // read x section and fill in y-direction final Array array = arrayCache.get(variableName); final int[] shape = array.getShape(); - final Number fillValue = NetCDFUtils.getDefaultFillValue(DataType.DOUBLE, true); + final double fillValue = NetCDFUtils.getDefaultFillValue(DataType.DOUBLE, true).doubleValue(); final int sectionWidth = interval.getX(); final int sectionHeight = interval.getY(); @@ -181,7 +189,7 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl if (x >= 0 && x < shape[0]) { value = array.getDouble(x); } else { - value = fillValue.doubleValue(); + value = fillValue; } for (int y = 0; y < sectionHeight; y++) { @@ -192,39 +200,81 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl } return resultArray; - } else if (variables2D.contains(variableName)) { - final Array array = arrayCache.get(variableName); - final Number fillValue = arrayCache.getNumberAttributeValue(NetCDFUtils.CF_FILL_VALUE_NAME, variableName); - return RawDataReader.read(centerX, centerY, interval, fillValue, array, getProductSize()); - } else { - // extract layer indices and NetCDF file variable name from variable name - final ArrayInfo arrayInfo = extractArrayInfo(variableName); - - final Array array = arrayCache.get(arrayInfo.ncVarName); - final Number fillValue = arrayCache.getNumberAttributeValue(NetCDFUtils.CF_FILL_VALUE_NAME, arrayInfo.ncVarName); - - final int[] origin; - final int[] shape; - if (arrayInfo.freqIdx >= 0) { - origin = new int[4]; - origin[2] = arrayInfo.lookIdx; - origin[3] = arrayInfo.freqIdx; - - shape = array.getShape(); - shape[2] = 1; - shape[3] = 1; + } + final int winWith = interval.getX(); + final int winHeight = interval.getY(); + final int offsetX = centerX - winWith / 2; + final int offsetY = centerY - winHeight / 2; + final boolean isWindowInside = RawDataReader.isWindowInside(offsetX, offsetY, winWith, winHeight, productWidth, productHeight); + + if (variables2D.contains(variableName)) { + final Variable variable = netcdfFile.findVariable(variableName); + if (isWindowInside) { + return variable.read(new int[]{offsetY, offsetX}, new int[]{winHeight, winWith}); } else { - origin = new int[4]; - origin[0] = arrayInfo.polIdx; - origin[1] = arrayInfo.lookIdx; - - shape = array.getShape(); - shape[0] = 1; - shape[1] = 1; + final Rectangle2D insideWindow = RawDataReader.getInsideWindow(offsetX, offsetY, winWith, winHeight, productWidth, productHeight); + final int[] origin = {(int) insideWindow.getY(), (int) insideWindow.getX()}; + final int[] shape = {(int) insideWindow.getHeight(), (int) insideWindow.getWidth()}; + final Array insideRead = variable.read(origin, shape); + final Number fillValue = NetCDFUtils.getFillValue(variable); + final Array array = NetCDFUtils.create(insideRead.getDataType(), new int[]{winHeight, winWith}, fillValue); + final int targetOffsY = Math.min(offsetY, 0); + final int targetOffsX = Math.min(offsetX, 0); + PartialDataCopier.copy(new int[]{targetOffsY, targetOffsX}, insideRead, array); + return array; } + } - final Array section = array.section(origin, shape).copy(); - return RawDataReader.read(centerX, centerY, interval, fillValue, section, getProductSize()); + // extract layer indices and NetCDF file variable name from variable name + final ArrayInfo arrayInfo = extractArrayInfo(variableName); + final Variable variable = netcdfFile.findVariable(arrayInfo.ncVarName); + final int[] shape = variable.getShape(); + final int[] origin = new int[4]; + final int xDimPos; + final int yDimPos; + final int lookPos; + final int freqPolPos; + if (arrayInfo.freqIdx >= 0) { + yDimPos = 0; + xDimPos = 1; + lookPos = 2; + freqPolPos = 3; + } else { + freqPolPos = 0; + lookPos = 1; + yDimPos = 2; + xDimPos = 3; + } + if (arrayInfo.freqIdx >= 0) { + origin[lookPos] = arrayInfo.lookIdx; + origin[freqPolPos] = arrayInfo.freqIdx; + shape[lookPos] = 1; + shape[freqPolPos] = 1; + } else { + origin[freqPolPos] = arrayInfo.polIdx; + origin[lookPos] = arrayInfo.lookIdx; + shape[freqPolPos] = 1; + shape[lookPos] = 1; + } + if (isWindowInside) { + origin[xDimPos] = offsetX; + origin[yDimPos] = offsetY; + shape[xDimPos] = winWith; + shape[yDimPos] = winHeight; + return variable.read(origin, shape).reduce(); + } else { + final Rectangle2D insideWindow = RawDataReader.getInsideWindow(offsetX, offsetY, winWith, winHeight, productWidth, productHeight); + origin[xDimPos] = (int) insideWindow.getX(); + origin[yDimPos] = (int) insideWindow.getY(); + shape[xDimPos] = (int) insideWindow.getWidth(); + shape[yDimPos] = (int) insideWindow.getHeight(); + final Array insideRead = variable.read(origin, shape).reduce(); + final Number fillValue = NetCDFUtils.getFillValue(variable); + final Array array = NetCDFUtils.create(insideRead.getDataType(), new int[]{winHeight, winWith}, fillValue); + final int targetOffsY = Math.min(offsetY, 0); + final int targetOffsX = Math.min(offsetX, 0); + PartialDataCopier.copy(new int[]{targetOffsY, targetOffsX}, insideRead, array); + return array; } } @@ -237,8 +287,10 @@ public Array readScaled(int centerX, int centerY, Interval interval, String vari } final ArrayInfo arrayInfo = extractArrayInfo(variableName); - final double scaleFactor = arrayCache.getNumberAttributeValue("scale_factor", arrayInfo.ncVarName).doubleValue(); - final double offset = arrayCache.getNumberAttributeValue("add_offset", arrayInfo.ncVarName).doubleValue(); + final Variable variable = netcdfFile.findVariable(arrayInfo.ncVarName); + final AttributeContainer attributes = variable.attributes(); + final double scaleFactor = attributes.findAttributeDouble("scale_factor", 1.0); + final double offset = attributes.findAttributeDouble("add_offset", 0.0); final MAMath.ScaleOffset scaleOffset = new MAMath.ScaleOffset(scaleFactor, offset); return MAMath.convert2Unpacked(rawArray, scaleOffset); } @@ -273,10 +325,14 @@ public List getVariables() throws InvalidRangeException, IOException { @Override public Dimension getProductSize() throws IOException { - final Array longitudes = arrayCache.get("longitude"); - final int[] shape = longitudes.getShape(); + return productSize; + } - return new Dimension("size", shape[1], shape[0]); + public void initProductSize() { + final int[] shape = netcdfFile.findVariable("longitude").getShape(); + productWidth = shape[1]; + productHeight = shape[0]; + productSize = new Dimension("size", productWidth, productHeight); } @Override diff --git a/core/src/main/java/com/bc/fiduceo/util/JDomUtils.java b/core/src/main/java/com/bc/fiduceo/util/JDomUtils.java index f52aa1d57..0ad3bca8e 100644 --- a/core/src/main/java/com/bc/fiduceo/util/JDomUtils.java +++ b/core/src/main/java/com/bc/fiduceo/util/JDomUtils.java @@ -16,9 +16,9 @@ */ package com.bc.fiduceo.util; -import org.jdom.Attribute; -import org.jdom.Document; -import org.jdom.Element; +import org.jdom2.Attribute; +import org.jdom2.Document; +import org.jdom2.Element; import java.util.List; @@ -34,16 +34,15 @@ public class JDomUtils { public static Attribute getMandatoryAttribute(final Element element, final String name) { final Attribute attribute = element.getAttribute(name); - String elementName = element.getName(); if (attribute == null) { - throw new RuntimeException(ATTRIBUTE + " '" + name + "' expected at element '" + elementName + "'"); + throw new RuntimeException(ATTRIBUTE + " '" + name + "' expected at element '" + element.getName() + "'"); } return attribute; } public static String getMandatoryText(final Element element) { final String textTrim = element.getTextTrim(); - if (textTrim.length() == 0) { + if (textTrim.isEmpty()) { throw new RuntimeException(VALUE + " of element '" + element.getName() + "' expected"); } return textTrim; diff --git a/core/src/main/java/com/bc/fiduceo/util/NetCDFUtils.java b/core/src/main/java/com/bc/fiduceo/util/NetCDFUtils.java index ea4b30137..95440e423 100644 --- a/core/src/main/java/com/bc/fiduceo/util/NetCDFUtils.java +++ b/core/src/main/java/com/bc/fiduceo/util/NetCDFUtils.java @@ -42,6 +42,7 @@ public class NetCDFUtils { public static final String CF_VALID_RANGE_NAME = "valid_range"; public static final String CF_UNITS_NAME = "units"; public static final String CF_STANDARD_NAME = "standard_name"; + public static final String CF_ANCILLARY_VARIABLES_NAME = "ancillary_variables"; public static final String CF_LONG_NAME = "long_name"; public static final String CF_FLAG_MEANINGS_NAME = "flag_meanings"; public static final String CF_FLAG_MASKS_NAME = "flag_masks"; @@ -357,12 +358,16 @@ public static Array scaleIfNecessary(Variable variable, Array array) { final double scaleFactor = NetCDFUtils.getScaleFactor(variable); final double offset = NetCDFUtils.getOffset(variable); if (ReaderUtils.mustScale(scaleFactor, offset)) { - final MAMath.ScaleOffset scaleOffset = new MAMath.ScaleOffset(scaleFactor, offset); - array = MAMath.convert2Unpacked(array, scaleOffset); + array = scale(array, scaleFactor, offset); } return array; } + public static Array scale(Array array, double scaleFactor, double offset) { + final MAMath.ScaleOffset scaleOffset = new MAMath.ScaleOffset(scaleFactor, offset); + return MAMath.convert2Unpacked(array, scaleOffset); + } + public static Attribute getGlobalAttributeSafe(String attributeName, NetcdfFile netcdfFile) { final Attribute globalAttribute = netcdfFile.findGlobalAttribute(attributeName); if (globalAttribute == null) { diff --git a/core/src/main/java/com/bc/fiduceo/util/VariableProxy.java b/core/src/main/java/com/bc/fiduceo/util/VariableProxy.java index 65412fe55..76d95f4bc 100644 --- a/core/src/main/java/com/bc/fiduceo/util/VariableProxy.java +++ b/core/src/main/java/com/bc/fiduceo/util/VariableProxy.java @@ -81,4 +81,9 @@ public void setShape(int[] shape) { public int[] getShape() { return this.shape; } + + @Override + public String toString() { + return "VariableProxy{ " + dataType + " " + name + "'(...)\r\n" + attributes + " }"; + } } diff --git a/core/src/main/resources/META-INF/services/com.bc.fiduceo.reader.ReaderPlugin b/core/src/main/resources/META-INF/services/com.bc.fiduceo.reader.ReaderPlugin index 7263f2868..45c183f64 100644 --- a/core/src/main/resources/META-INF/services/com.bc.fiduceo.reader.ReaderPlugin +++ b/core/src/main/resources/META-INF/services/com.bc.fiduceo.reader.ReaderPlugin @@ -1,4 +1,5 @@ com.bc.fiduceo.reader.amsu_mhs.AMSUB_MHS_L1C_ReaderPlugin +com.bc.fiduceo.reader.amsu_mhs.MHS_L1B_ReaderPlugin com.bc.fiduceo.reader.airs.AIRS_L1B_ReaderPlugin com.bc.fiduceo.reader.hirs.HIRS_L1C_ReaderPlugin com.bc.fiduceo.reader.iasi.IASI_ReaderPlugin @@ -30,6 +31,7 @@ com.bc.fiduceo.reader.insitu.sirds_sst.SirdsCtdInsituReaderPlugin com.bc.fiduceo.reader.insitu.sirds_sst.SirdsDrifterCmemsInsituReaderPlugin com.bc.fiduceo.reader.insitu.sirds_sst.SirdsDrifterInsituReaderPlugin com.bc.fiduceo.reader.insitu.sirds_sst.SirdsGtmbaInsituReaderPlugin +com.bc.fiduceo.reader.insitu.sirds_sst.SirdsGtmba2InsituReaderPlugin com.bc.fiduceo.reader.insitu.sirds_sst.SirdsMbtInsituReaderPlugin com.bc.fiduceo.reader.insitu.sirds_sst.SirdsMooringInsituReaderPlugin com.bc.fiduceo.reader.insitu.sirds_sst.SirdsShipInsituReaderPlugin @@ -38,6 +40,11 @@ com.bc.fiduceo.reader.smos.SmosL1CDailyGriddedReaderPlugin com.bc.fiduceo.reader.insitu.sic_cci.ANTXXXISicInsituReaderPlugin com.bc.fiduceo.reader.insitu.sic_cci.DMISIC0SicInsituReaderPlugin com.bc.fiduceo.reader.insitu.sic_cci.DTUSIC1SicInsituReaderPlugin +com.bc.fiduceo.reader.smap.SmapReaderPluginForLook +com.bc.fiduceo.reader.smap.SmapReaderPluginAftLook com.bc.fiduceo.reader.windsat.WindsatReaderPlugin com.bc.fiduceo.reader.insitu.ndbc.NdbcCWReaderPlugin -com.bc.fiduceo.reader.insitu.ndbc.NdbcSMReaderPlugin \ No newline at end of file +com.bc.fiduceo.reader.insitu.ndbc.NdbcSMReaderPlugin +com.bc.fiduceo.reader.insitu.tao.TaoReaderPlugin +com.bc.fiduceo.reader.amsu_mhs.AMSUA_L1B_ReaderPlugin +com.bc.fiduceo.reader.insitu.generic.GbovReaderPlugin \ No newline at end of file diff --git a/core/src/main/resources/com/bc/fiduceo/qc/bluemarble-2048.png b/core/src/main/resources/com/bc/fiduceo/qc/bluemarble-2048.png new file mode 100644 index 000000000..2ee151841 Binary files /dev/null and b/core/src/main/resources/com/bc/fiduceo/qc/bluemarble-2048.png differ diff --git a/core/src/main/resources/com/bc/fiduceo/reader/amsu_mhs/nat/AMSUA_L1B/variables.json b/core/src/main/resources/com/bc/fiduceo/reader/amsu_mhs/nat/AMSUA_L1B/variables.json new file mode 100644 index 000000000..01f874ac5 --- /dev/null +++ b/core/src/main/resources/com/bc/fiduceo/reader/amsu_mhs/nat/AMSUA_L1B/variables.json @@ -0,0 +1,82 @@ +{ + "variables": { + "SCENE_RADIANCE_*": { + "data_type": "integer4", + "offset": 22, + "stride": 15, + "scale_factor": 0.0000001, + "units" : "mW/m2/sr/cm-1", + "standard_name": "toa_radiance" + }, + "solar_zenith_angle": { + "data_type": "integer2", + "offset": 1842, + "stride": 4, + "scale_factor": 0.01, + "units" : "degree", + "standard_name": "solar_zenith_angle" + }, + "satellite_zenith_angle": { + "data_type": "integer2", + "offset": 1844, + "stride": 4, + "scale_factor": 0.01, + "units" : "degree", + "standard_name": "platform_zenith_angle" + }, + "solar_azimuth_angle": { + "data_type": "integer2", + "offset": 1846, + "stride": 4, + "scale_factor": 0.01, + "units" : "degree", + "standard_name": "solar_azimuth_angle" + }, + "satellite_azimuth_angle": { + "data_type": "integer2", + "offset": 1848, + "stride": 4, + "scale_factor": 0.01, + "units" : "degree", + "standard_name": "platform_azimuth_angle" + }, + "latitude": { + "data_type": "integer4", + "offset": 2082, + "stride": 2, + "scale_factor": 0.0001, + "units" : "degree", + "standard_name": "latitude" + }, + "longitude": { + "data_type": "integer4", + "offset": 2086, + "stride": 2, + "scale_factor": 0.0001, + "units" : "degree", + "standard_name": "longitude" + }, + "SURFACE_PROPERTIES": { + "data_type": "integer2", + "offset": 2322, + "stride": 1, + "flag_values": "0, 1, 2", + "flag_meanings": "water mixed_coast land" + }, + "TERRAIN_ELEVATION": { + "data_type": "integer2", + "offset": 2382, + "stride": 1, + "units" : "m", + "standard_name": "height_above_mean_sea_level" + }, + "TIME_ATTITUDE": { + "data_type": "u-integer4", + "data_layout": "VECTOR", + "offset": 1824, + "stride": 1, + "units" : "s", + "standard_name": "time" + } + } +} \ No newline at end of file diff --git a/core/src/main/resources/com/bc/fiduceo/reader/amsu_mhs/nat/MHS_L1B/variables.json b/core/src/main/resources/com/bc/fiduceo/reader/amsu_mhs/nat/MHS_L1B/variables.json new file mode 100644 index 000000000..2b441927f --- /dev/null +++ b/core/src/main/resources/com/bc/fiduceo/reader/amsu_mhs/nat/MHS_L1B/variables.json @@ -0,0 +1,86 @@ +{ + "variables": { + "SCENE_RADIANCES_*": { + "data_type": "integer4", + "offset": 83, + "stride": 5, + "scale_factor": 0.0000001, + "units": "mW/m2/sr/cm-1", + "standard_name": "toa_radiance" + }, + "FOV_DATA_QUALITY": { + "data_type": "integer4", + "offset": 1883, + "stride": 1, + "standard_name": "quality_flag" + }, + "solar_zenith_angle": { + "data_type": "integer2", + "offset": 2598, + "stride": 4, + "scale_factor": 0.01, + "units" : "degree", + "standard_name": "solar_zenith_angle" + }, + "satellite_zenith_angle": { + "data_type": "integer2", + "offset": 2600, + "stride": 4, + "scale_factor": 0.01, + "units" : "degree", + "standard_name": "platform_zenith_angle" + }, + "solar_azimuth_angle": { + "data_type": "integer2", + "offset": 2602, + "stride": 4, + "scale_factor": 0.01, + "units" : "degree", + "standard_name": "solar_azimuth_angle" + }, + "satellite_azimuth_angle": { + "data_type": "integer2", + "offset": 2604, + "stride": 4, + "scale_factor": 0.01, + "units" : "degree", + "standard_name": "platform_azimuth_angle" + }, + "latitude": { + "data_type": "integer4", + "offset": 3318, + "stride": 2, + "scale_factor": 0.0001, + "units": "degree" + }, + "longitude": { + "data_type": "integer4", + "offset": 3322, + "stride": 2, + "scale_factor": 0.0001, + "units": "degree" + }, + "SURFACE_PROPERTIES": { + "data_type": "byte", + "offset": 4038, + "stride": 1, + "flag_values": "0, 1, 2", + "flag_meanings": "water mixed_coast land" + }, + "TERRAIN_ELEVATION": { + "data_type": "integer2", + "offset": 4128, + "stride": 1, + "units" : "m", + "standard_name": "height_above_mean_sea_level" + }, + "TIME_ATTITUDE": { + "data_type": "u-integer4", + "data_layout": "VECTOR", + "offset": 2580, + "stride": 1, + "units" : "s", + "standard_name": "time" + } + } +} \ No newline at end of file diff --git a/core/src/main/resources/com/bc/fiduceo/reader/insitu/generic/GBOV_config.json b/core/src/main/resources/com/bc/fiduceo/reader/insitu/generic/GBOV_config.json new file mode 100644 index 000000000..b21a94cb9 --- /dev/null +++ b/core/src/main/resources/com/bc/fiduceo/reader/insitu/generic/GBOV_config.json @@ -0,0 +1,869 @@ +{ + "name": "GBOV", + "delimiter": ";", + "comment_char": "#", + "regex": "^GBOV__(.*?)__(.*?)__([0-9]{8}T[0-9]{6}Z)__([0-9]{8}T[0-9]{6}Z)\\.csv$", + "longitude_name": "Lon_IS", + "latitude_name": "Lat_IS", + "time_name": "TIME_IS", + "time_format": "yyyyMMdd'T'HHmmssX", + "location_from_station_database": true, + "variables": [ + { "name": "TIME_IS", "type": "int", "units": "seconds since 1970-01-01", "cf_standard": "time" }, + { "name": "FIPAR_down", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Understory fraction of intercepted photosynthetically active radiation"}, + { "name": "FIPAR_down_err", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Uncertainty of understory fraction of intercepted photosynthetically active radiation"}, + { "name": "FIPAR_total", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Total fraction of intercepted photosynthetically active radiation", "cf_standard": "fraction_of_surface_downwelling_photosynthetic_radiative_flux_intercepted_by_vegetation"}, + { "name": "FIPAR_total_err", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Uncertainty of total fraction of intercepted photosynthetically active radiation"}, + { "name": "FIPAR_up", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Overstory fraction of intercepted photosynthetically active radiation"}, + { "name": "FIPAR_up_err", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Uncertainty of overstory fraction of intercepted photosynthetically active radiation"}, + { "name": "RM6_down_flag", "type": "short", "fill_value": -999, "units": "1", "long_name": "Quality flag for understory FIPAR measurement (0 = valid, >1 = suspect)", "ancillary_variables": "down_flag" }, + { "name": "RM6_up_flag", "type": "short", "fill_value": -999, "units": "1", "long_name": "Quality flag for overstory FIPAR measurement (0 = valid, >1 = suspect)", "ancillary_variables": "up_flag" }, + { "name": "Clumping_Miller", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Clumping index (Miller method)"}, + { "name": "Clumping_Miller_err", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Uncertainty of clumping index (Miller method)"}, + { "name": "Clumping_Warren", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Clumping index (Warren-Wilson method)"}, + { "name": "Clumping_Warren_err", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Uncertainty of clumping index (Warren-Wilson method)"}, + { "name": "LAI_Miller_down", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Understory true leaf area index (Miller method)"}, + { "name": "LAI_Miller_down_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of understory true leaf area index (Miller method)"}, + { "name": "LAI_Miller_up", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Overstory true leaf area index (Miller method)"}, + { "name": "LAI_Miller_up_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of overstory true leaf area index (Miller method)"}, + { "name": "LAI_Warren_down", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Understory true leaf area index (Warren-Wilson method)"}, + { "name": "LAI_Warren_down_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of understory true leaf area index (Warren-Wilson method)"}, + { "name": "LAI_Warren_up", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Overstory true leaf area index (Warren-Wilson method)"}, + { "name": "LAI_Warren_up_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of overstory true leaf area index (Warren-Wilson method)"}, + { "name": "LAI_down", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Understory leaf area index (aggregate)"}, + { "name": "LAI_total_Miller", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Total leaf area index (Miller method)", "cf_standard": "leaf_area_index "}, + { "name": "LAI_total_Warren", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Total leaf area index (Warren-Wilson method)", "cf_standard": "leaf_area_index "}, + { "name": "LAIe_Miller_down", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Understory effective leaf area index (Miller method)"}, + { "name": "LAIe_Miller_down_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of understory effective leaf area index (Miller method)"}, + { "name": "LAIe_Miller_up", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Overstory effective leaf area index (Miller method)"}, + { "name": "LAIe_Miller_up_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of overstory effective leaf area index (Miller method)"}, + { "name": "LAIe_Warren_down", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Understory effective leaf area index (Warren-Wilson method)"}, + { "name": "LAIe_Warren_down_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of understory effective leaf area index (Warren-Wilson method)"}, + { "name": "LAIe_Warren_up", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Overstory effective leaf area index (Warren-Wilson method)"}, + { "name": "LAIe_Warren_up_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of overstory effective leaf area index (Warren-Wilson method)"}, + { "name": "PAI_Miller", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Plant area index (Miller method)"}, + { "name": "PAI_Miller_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of plant area index (Miller method)"}, + { "name": "PAI_Warren", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Plant area index (Warren-Wilson method)"}, + { "name": "PAI_Warren_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of plant area index (Warren-Wilson method)"}, + { "name": "PAIe_Miller", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Effective plant area index (Miller method)"}, + { "name": "PAIe_Miller_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of effective plant area index (Miller method)"}, + { "name": "PAIe_Warren", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Effective plant area index (Warren-Wilson method)"}, + { "name": "PAIe_Warren_err", "type": "float", "fill_value": -999.0, "units": "m² m⁻²", "long_name": "Uncertainty of effective plant area index (Warren-Wilson method)"}, + { "name": "clumping_Miller_down", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Understory clumping index (Miller method)"}, + { "name": "clumping_Miller_down_err", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Uncertainty of understory clumping index (Miller method)"}, + { "name": "clumping_Miller_up", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Overstory clumping index (Miller method)"}, + { "name": "clumping_Miller_up_err", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Uncertainty of overstory clumping index (Miller method)"}, + { "name": "clumping_Warren_down", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Understory clumping index (Warren-Wilson method)"}, + { "name": "clumping_Warren_down_err", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Uncertainty of understory clumping index (Warren-Wilson method)"}, + { "name": "clumping_Warren_up", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Overstory clumping index (Warren-Wilson method)"}, + { "name": "clumping_Warren_up_err", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Uncertainty of overstory clumping index (Warren-Wilson method)"}, + { "name": "RM7_down_flag", "type": "short", "fill_value": -999, "units": "1", "long_name": "Quality flag for understory LAI/PAI measurements (0 = valid, >1 = suspect)", "ancillary_variables": "down_flag" }, + { "name": "RM7_up_flag", "type": "short", "fill_value": -999, "units": "1", "long_name": "Quality flag for overstory LAI/PAI measurements (0 = valid, >1 = suspect)", "ancillary_variables": "up_flag" }, + { "name": "LSE", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Land surface longwave emissivity", "cf_standard": "surface_longwave_emissivity" }, + { "name": "LSE_STD", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Standard deviation of land surface longwave emissivity"}, + { "name": "LSR", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Spectral land surface radiance"}, + { "name": "LSR_STD", "type": "float", "fill_value": -999.0, "units": "1", "long_name": "Standard deviation of spectral land surface radiance"}, + { "name": "QC_LSE", "type": "short", "fill_value": -999, "units": "1", "long_name": "Quality flag for land surface emissivity (0 = valid, 1 = suspect)"}, + { "name": "QC_LSR", "type": "short", "fill_value": -999, "units": "1", "long_name": "Quality flag for spectral land surface radiance (0 = valid, 1 = suspect)" }, + { "name": "LST", "type": "float", "fill_value": -999.0, "units": "K", "long_name": "Land surface (skin) temperature", "cf_standard": "surface_temperature_where_land" }, + { "name": "LST_STD", "type": "float", "fill_value": -999.0, "units": "K", "long_name": "Standard deviation of land surface (skin) temperature" }, + { "name": "QC_LST", "type": "short", "fill_value": -999, "units": "1", "long_name": "Quality flag for land surface temperature (0 = valid, 1 = suspect)" }, + { "name": "QC_SM_5", "type": "short", "fill_value": -999, "units": "1", "long_name": "Quality flag for soil moisture at 5 cm depth (0 = valid, 1 = suspect)" }, + { "name": "SM_5", "type": "double", "fill_value": -999.0, "units": "m³ m⁻³", "long_name": "Volumetric soil moisture at 5 cm depth", "cf_standard": "volume_fraction_of_condensed_water_in_soil" } + ], + "station_database": { + "primary_id": "site", + "secondary_id": "station", + "variables": [ + { "origin": "s","name": "site", "type": "string", "units": "1", "long_name": "Insitu data measurement site"}, + { "origin": "s","name": "station", "type": "string", "units": "1", "cf_standard": "platform_name", "long_name": "Insitu data measurement station"}, + { "origin": "s","name": "elevation", "type": "float", "units": "m", "cf_standard": "height_above_mean_sea_level", "long_name": "Height_above_X means the vertical distance above the named surface X. \"Mean sea level\" means the time mean of sea surface elevation at a given location over an arbitrary period sufficient to eliminate the tidal signals."}, + { "origin": "s","name": "IGBP_class", "type": "string", "units": "1", "long_name": "International Geosphere–Biosphere Programme ecosystem surface classification"}, + { "origin": "s","name": "Lat_IS", "type": "float", "units": "degree_north", "cf_standard": "latitude", "long_name": "Latitude is positive northward; its units of degree_north (or equivalent) indicate this explicitly. In a latitude-longitude system defined with respect to a rotated North Pole, the standard name of grid_latitude should be used instead of latitude. Grid latitude is positive in the grid-northward direction, but its units should be plain degree."}, + { "origin": "s","name": "Lon_IS", "type": "float", "units": "degree_east", "cf_standard": "longitude", "long_name": "Longitude is positive eastward; its units of degree_east (or equivalent) indicate this explicitly. In a latitude-longitude system defined with respect to a rotated North Pole, the standard name of grid_longitude should be used instead of longitude. Grid longitude is positive in the grid-eastward direction, but its units should be plain degree."} + ], + "stations": [ + ["Cumberland Plain","CumberlandPlain",25,"Evergreen Broadleaf",-33.615,150.723], + ["Steigerwaldt Land Services","STEI_016",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_005",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_015",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_004",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_058",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_014",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_003",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_046",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_013",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_002",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_009",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_019",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_008",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_007",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_017",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_006",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_012",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_001",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_055",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_022",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_011",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_021",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Steigerwaldt Land Services","STEI_010",481,"Deciduous Broadleaf",45.5089416503906,-89.5863723754883], + ["Condom","Smos3",174,"Croplands",43.9744,0.3361], + ["Dead Lake","DELA_018",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_007",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_008",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_016",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_005",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_017",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_006",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_014",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_003",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_015",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_004",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_012",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_001",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_046",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_013",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_002",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_010",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_044",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_011",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_053",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_020",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_051",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Dead Lake","DELA_009",22,"Deciduous Broadleaf",32.54172,-87.80389], + ["Wustebach","Wustebach",610,"Evergreen Needleleaf Forest",50.50493,6.3309627], + ["Southern Great Plains","SouthernGreatPlains",318,"Croplands",36.60575,-97.48876], + ["Gobabeb","Gobabeb",407,"Bare soil and Rocks",-23.5618361,15.0413138888889], + ["Niwot Ridge Mountain Research Station","NIWO_060",3050,"Evergreen Needleleaf",40.0543,-105.58245], + ["Niwot Ridge Mountain Research Station","NIWO_048",3050,"Evergreen Needleleaf",40.0543,-105.58245], + ["Niwot Ridge Mountain Research Station","NIWO_056",3050,"Evergreen Needleleaf",40.0543,-105.58245], + ["Niwot Ridge Mountain Research Station","NIWO_054",3050,"Evergreen Needleleaf",40.0543,-105.58245], + ["Bartlett Experimental Forest","BART_018",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_007",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_028",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_006",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_034",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_023",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_012",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_001",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_011",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_010",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_027",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_016",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_005",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_026",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_015",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_004",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_047",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_025",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_003",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_024",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_013",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_002",232,"Mixed Forest",44.063901,-71.287308], + ["Bartlett Experimental Forest","BART_041",232,"Mixed Forest",44.063901,-71.287308], + ["Pezenas","Smos10",30,"Croplands",43.43833,3.40333], + ["North Sterling","STER_032",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_033",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_031",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_009",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_014",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_026",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_034",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_035",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_029",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_018",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_008",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_027",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_005",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_028",1364,"Grasslands",40.461952,-103.02934], + ["North Sterling","STER_006",1364,"Grasslands",40.461952,-103.02934], + ["Tateno","Tateno",25,"Urban and Built-up",36.05918,140.12444], + ["Watkinsville","SSE",216,"Grassland",33.78,-83.39], + ["Wicken Fen","WickenFen",46,"Croplands",52.3094,0.27972], + ["Desert Rock","Desert_Rock_NV",1007,"Open Shrublands",36.62418,-116.0199], + ["Hohes Holz","DE-HoH_CP03",129,"Deciduous Broadleaf Forest",52.08656,11.22235], + ["Hohes Holz","DE-HoH_CP04",129,"Deciduous Broadleaf Forest",52.08656,11.22235], + ["Hohes Holz","DE-HoH_CP01",129,"Deciduous Broadleaf Forest",52.08656,11.22235], + ["Hohes Holz","DE-HoH_CP02",129,"Deciduous Broadleaf Forest",52.08656,11.22235], + ["Hohes Holz","DE-HoH_CP05",129,"Deciduous Broadleaf Forest",52.08656,11.22235], + ["Great Western Woodland","GreatWesternWoodland",448,"Open Shrublands",-30.192,120.654], + ["Jones Ecological Research Center","JERC_054",44,"Evergreen Needleleaf",31.1948394775391,-84.468777], + ["Jones Ecological Research Center","JERC_050",44,"Evergreen Needleleaf",31.1948394775391,-84.468777], + ["Jones Ecological Research Center","JERC_062",44,"Evergreen Needleleaf",31.1948394775391,-84.468777], + ["Jones Ecological Research Center","JERC_060",44,"Evergreen Needleleaf",31.1948394775391,-84.468777], + ["Yanco","y9",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","Yanco",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y1",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y11",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y2",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y10",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y3",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y13",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y4",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y12",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y5",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y6",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y7",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Yanco","y8",119,"Grasslands",-35.0053482055664,146.309875488281], + ["Tibetian Plateau (Naqu)","NAQU2",4591,"Grasslands",31.31,91.82], + ["Tibetian Plateau (Naqu)","NAQU1",4591,"Grasslands",31.31,91.82], + ["Tibetian Plateau (Naqu)","NAQU4",4591,"Grasslands",31.31,91.82], + ["Tibetian Plateau (Naqu)","NAQU3",4591,"Grasslands",31.31,91.82], + ["Tibetian Plateau (Naqu)","NAQU5",4591,"Grasslands",31.31,91.82], + ["Soaproot Saddle","SOAP_019",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_008",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_009",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_015",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_004",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_016",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_005",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_017",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_006",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_018",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_007",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_011",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_012",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_001",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_013",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_002",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_058",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_047",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_003",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_020",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_054",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_021",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Soaproot Saddle","SOAP_010",905,"Evergreen Needleleaf Forest",37.03337,-119.26219], + ["Onaqui Ault","ONAQ_009",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_019",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_008",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_018",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_007",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_017",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_016",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_005",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_015",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_004",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_069",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_025",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_014",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_003",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_002",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_023",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_012",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_011",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_021",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_010",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_020",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_051",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Onaqui Ault","ONAQ_060",1685,"Open Shrublands",40.1775894165039,-112.452438354492], + ["Mouthoumet","Smos7",538,"Croplands",42.96,2.53], + ["Kyeamba","k2",437,"Croplands",-35.4932,147.559], + ["Kyeamba","k5",437,"Croplands",-35.4932,147.559], + ["Kyeamba","k11",437,"Croplands",-35.4932,147.559], + ["Kyeamba","k6",437,"Croplands",-35.4932,147.559], + ["Kyeamba","k10",437,"Croplands",-35.4932,147.559], + ["Kyeamba","k7",437,"Croplands",-35.4932,147.559], + ["Kyeamba","k13",437,"Croplands",-35.4932,147.559], + ["Kyeamba","k12",437,"Croplands",-35.4932,147.559], + ["Kyeamba","k14",437,"Croplands",-35.4932,147.559], + ["Sioux Falls SurfRad","Sioux_Falls_SD",473,"Croplands",43.7340316772461,-96.62331], + ["Barrow","Barrow",11,"Snow and Ice",71.3231,-156.61052], + ["Saskatchewan","SK1",574,"Croplands",51.3347,-106.5647], + ["Saskatchewan","SK3",574,"Croplands",51.3347,-106.5647], + ["Saskatchewan","SK2",574,"Croplands",51.3347,-106.5647], + ["Saskatchewan","SK4",574,"Croplands",51.3347,-106.5647], + ["Valencia Anchor Station","v6",813,"Cropland Mosaics",39.5707206726074,-1.28822004795074], + ["Valencia Anchor Station","v7",813,"Cropland Mosaics",39.5707206726074,-1.28822004795074], + ["Valencia Anchor Station","v8",813,"Cropland Mosaics",39.5707206726074,-1.28822004795074], + ["Valencia Anchor Station","v9",813,"Cropland Mosaics",39.5707206726074,-1.28822004795074], + ["Valencia Anchor Station","v1",813,"Cropland Mosaics",39.5707206726074,-1.28822004795074], + ["Valencia Anchor Station","v2",813,"Cropland Mosaics",39.5707206726074,-1.28822004795074], + ["Valencia Anchor Station","v3",813,"Cropland Mosaics",39.5707206726074,-1.28822004795074], + ["Valencia Anchor Station","v4",813,"Cropland Mosaics",39.5707206726074,-1.28822004795074], + ["Valencia Anchor Station","v5",813,"Cropland Mosaics",39.5707206726074,-1.28822004795074], + ["Yosemite Village","YosemiteVillage",2081,"Evergreen Needleleaf",37.7588889,-119.8211], + ["Danube mega farme 1 Calarasi","Ca1",20,"Croplands",44.2057,27.3385], + ["Table Mountain","Boulder_CO",1689,"Bare soil and Rocks",40.1249809265137,-105.236801147461], + ["Rock Springs","Penn_State_PA",376,"Deciduous Broadleaf",40.7201194763184,-77.9308471679688], + ["Manhattan","m1",347,"Grasslands",39.10274,-96.60974], + ["Manhattan","Manhattan",347,"Grasslands",39.10274,-96.60974], + ["Rosiorii de Vede","RdV1",102,"Croplands",44.10711,24.97872], + ["Manitoba","MB6",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB5",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB8",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB7",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB13",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB9",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB12",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB11",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB10",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB2",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB1",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB4",272,"Croplands",49.56222,-98.01916], + ["Manitoba","MB3",272,"Croplands",49.56222,-98.01916], + ["Cabauw","Cabauw",0,"Cropland Mosaics",51.9710006713867,4.92700004577637], + ["Moab","MOAB_061",1767,"Open Shrublands",38.24836,-109.38831], + ["Moab","MOAB_070",1767,"Open Shrublands",38.24836,-109.38831], + ["Moab","MOAB_049",1767,"Open Shrublands",38.24836,-109.38831], + ["Smithsonian Conservation Biology Institute","SCBI_008",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_010",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_063",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_062",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_014",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_003",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_035",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_013",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_002",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_034",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_012",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_033",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_011",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_018",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_007",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_017",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_006",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_049",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_038",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_016",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_005",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_015",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Smithsonian Conservation Biology Institute","SCBI_004",361,"Mixed Forest",38.8929214477539,-78.1395034790039], + ["Disney Wilderness Preserve","DSNY_061",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_053",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_066",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_055",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_011",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_021",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_010",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_013",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_002",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_034",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_012",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_001",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_015",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_004",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_025",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_014",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_003",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_017",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_006",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_016",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_005",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_008",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_007",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Disney Wilderness Preserve","DSNY_009",15,"Open Shrublands",28.1250400543213,-81.43625], + ["Sabres","Smos11",81,"Croplands",44.1475,-0.8456], + ["Hyltemossa","SE-Htm_CP04",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_CP03",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_CP02",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP11",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP12",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP01",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP20",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP10",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP15",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP04",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP16",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP05",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP13",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP02",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP14",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP03",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP19",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP08",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_CP01",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP09",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP17",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP06",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP18",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Hyltemossa","SE-Htm_SP07",41,"Evergreen Needleleaf Forest",56.09763,13.41897], + ["Warra Tall Eucalypt","WarraTallEucalypt",714,"Evergreen Broadleaf",-43.09499844,146.6544986], + ["Santa Rita","SRER_009",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_018",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_007",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_019",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_008",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_016",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_005",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_017",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_006",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_014",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_003",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_048",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_015",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_004",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_012",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_001",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_057",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_046",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_013",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_002",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_010",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_011",983,"Closed Shrublands",31.91068,-110.83549], + ["Santa Rita","SRER_020",983,"Closed Shrublands",31.91068,-110.83549], + ["Brasschaat","BE-Bra_CP04",16,"Mixed Forest",51.3075,4.5199], + ["Brasschaat","BE-Bra_CP03",16,"Mixed Forest",51.3075,4.5199], + ["Brasschaat","Brasschaat",16,"Mixed Forest",51.3075,4.5199], + ["Brasschaat","BE-Bra_CP02",16,"Mixed Forest",51.3075,4.5199], + ["Brasschaat","BE-Bra_CP01",16,"Mixed Forest",51.3075,4.5199], + ["Montaut","Smos6",295,"Croplands",43.1922,1.6436], + ["Calperum","Calperum",53,"Closed Shrublands",-34.0027008056641,140.587707519531], + ["Sioux Falls Uscrn","s1",497,"Croplands",43.73456,-96.62197], + ["Tereno","te2",610,"Deciduous Broadleaf",50.5049018859863,6.33109], + ["Tereno","te1",610,"Deciduous Broadleaf",50.5049018859863,6.33109], + ["Tereno","te4",610,"Deciduous Broadleaf",50.5049018859863,6.33109], + ["Tereno","te3",610,"Deciduous Broadleaf",50.5049018859863,6.33109], + ["Tereno","te5",610,"Deciduous Broadleaf",50.5049018859863,6.33109], + ["Des Moines","Des_Moines",281,"Croplands",41.5563,-93.2854995727539], + ["Narbonne","Smos8",112,"Evergreen Needleleaf Forest",43.15,2.9567], + ["Barnas","Smos1",480,"Evergreen Needleleaf Forest",44.666,4.15983], + ["Batesville","WNW",136,"Grassland",35.82,-91.78], + ["Peyrusse Grande","Smos9",245,"Croplands",43.6664,0.2217], + ["Tryon","E35",294,"Grassland",35.8615,-97.0695], + ["San Rossore 2","IT-SR2_SP12",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP01",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP13",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP02",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP14",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP03",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP15",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP04",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP20",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP10",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP11",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP09",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_CP01",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_CP02",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP16",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP05",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP17",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP06",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP18",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP07",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["San Rossore 2","IT-SR2_SP08",1,"Evergreen Needleleaf Forest",43.73202,10.29091], + ["Siikaneva","Siikaneva",160,"Permanent Wetland",61.83265,24.19285], + ["Imnavait Creek","i1",929,"Tundra",68.61654,-149.3032], + ["Central Plains Experimental Range","CPER_015",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_004",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_014",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_003",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_046",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_013",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_002",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_056",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_012",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_001",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_011",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_010",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_020",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_062",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_009",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_019",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_008",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_018",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_007",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_017",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_006",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_016",1648,"Grasslands",40.81555,-104.74566], + ["Central Plains Experimental Range","CPER_005",1648,"Grasslands",40.81555,-104.74566], + ["Lenoir Landing","LENO_014",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_003",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_015",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_004",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_067",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_012",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_001",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_013",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_002",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_021",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_010",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_022",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_011",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_063",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_064",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_009",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_018",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_007",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_008",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_016",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_005",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_017",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Lenoir Landing","LENO_006",13,"Deciduous Broadleaf Forest",31.853861,-88.161181], + ["Robson Creek Rainforest","RobsonCreekRainforest",680,"Mixed Forest",-17.117469,145.630139], + ["Lyndon B. Johnson National Grassland","CLBJ_002",272,"Deciduous Broadleaf Forest",33.40123,-97.57], + ["Lyndon B. Johnson National Grassland","CLBJ_003",272,"Deciduous Broadleaf Forest",33.40123,-97.57], + ["Lyndon B. Johnson National Grassland","CLBJ_001",272,"Deciduous Broadleaf Forest",33.40123,-97.57], + ["AQUI Parc Meteo","AQ1",108,"Croplands Mosaics",44.79,0.58], + ["Duke Forest","DukeForest",170,"Mixed Forest",35.9712,-79.0934], + ["Tibetian Plateau (Ngari)","NGARI2",4712,"Grasslands",31.33,91.854], + ["Tibetian Plateau (Ngari)","NGARI1",4712,"Grasslands",31.33,91.854], + ["Dry River","DryRiver",175,"Woody Savannas",-15.259,132.371], + ["Gingin","Gingin",111,"Woody Savannas",-31.3764,115.7139], + ["Tucson","W",164,"Open Shrublands",32.24,-111.17], + ["Goodwin Creek","Goodwin_Creek_MS",98,"Deciduous Broadleaf",34.25505,-89.8736], + ["Pu u Maka ala Natural Area Reserve","PUUM_039",1685,"Evergreen Broadleaf",19.55309,-155.31731], + ["Pu u Maka ala Natural Area Reserve","PUUM_031",1685,"Evergreen Broadleaf",19.55309,-155.31731], + ["Pu u Maka ala Natural Area Reserve","PUUM_041",1685,"Evergreen Broadleaf",19.55309,-155.31731], + ["Barlad","Br1",172,"Croplands",46.23313,27.64438], + ["Tibetian Plateau (Maqu)","MAQU",3725,"Croplands",33.68,101.89], + ["Hainich","ha1",430,"Mixed Forest",51.0792007446289,10.4522], + ["Hainich","ha4",430,"Mixed Forest",51.0792007446289,10.4522], + ["Hainich","ha6",430,"Mixed Forest",51.0792007446289,10.4522], + ["Hainich","ha5",430,"Mixed Forest",51.0792007446289,10.4522], + ["Tumbarumba","tu1",1200,"Evergreen Broadleaf",-35.65652,148.15163], + ["Ordway Swisher Biological Station","OSBS_009",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_008",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_007",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_006",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_038",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_016",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_005",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_048",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_015",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_004",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_025",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_003",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_024",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_002",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_023",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_012",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_001",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_022",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_011",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_010",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_030",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Ordway Swisher Biological Station","OSBS_051",45,"Evergreen Needleleaf",29.6839,-81.9934], + ["Konza Prairie Biological Station","KONA_073",323,"Croplands",39.110446,-96.612935], + ["Konza Prairie Biological Station","KONA_074",323,"Croplands",39.110446,-96.612935], + ["Konza Prairie Biological Station","KONA_071",323,"Croplands",39.110446,-96.612935], + ["Konza Prairie Biological Station","KONA_070",323,"Croplands",39.110446,-96.612935], + ["Guanica Forest","GUAN_003",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_013",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_002",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_012",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_001",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_055",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_011",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_054",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_010",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_020",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_052",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_009",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_019",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_008",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_018",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_007",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_017",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_006",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_016",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_005",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_015",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["Guanica Forest","GUAN_004",143,"Evergreen Broadleaf",17.9695491790771,-66.8686981201172], + ["University of Kansas Field Site","UKFS_015",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_004",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_016",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_005",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_017",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_006",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_018",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_007",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_019",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_008",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_009",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_053",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_010",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_011",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_012",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_001",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_013",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_002",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_058",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_047",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_025",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_014",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["University of Kansas Field Site","UKFS_003",322,"Deciduous Broadleaf Forest",39.040431,-95.19215], + ["Vielsalm","BE-Vie_SP14",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP03",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP13",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP02",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP12",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP01",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP11",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP10",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP20",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP09",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP19",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP08",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP18",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP07",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP17",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP06",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP16",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP05",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP15",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_SP04",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_CP01",483,"Mixed Forest",50.304962,5.998099], + ["Vielsalm","BE-Vie_CP02",483,"Mixed Forest",50.304962,5.998099], + ["Oak Ridge","ORNL_031",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_054",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_032",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_021",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_010",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_066",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_044",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_033",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_045",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_012",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_001",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_035",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_002",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_014",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_003",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_004",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_027",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_006",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_029",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_007",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_008",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_009",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Oak Ridge","ORNL_050",334,"Mixed Forest",35.9641189575195,-84.282600402832], + ["Inner Mongolia","IM1",1426,"Grasslands",45.74,106.26], + ["Alice Mulga","AliceMulga",583,"Open Shrublands",-22.283236,133.251096], + ["Hyytiala","FI-Hyy_SP17",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP18",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP19",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP13",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP14",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP15",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP16",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","h1",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_CP03",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_CP02",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_CP04",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_CP01",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP20",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP01",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP06",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP07",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP08",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP09",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP02",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP03",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP04",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP05",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP10",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP11",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Hyytiala","FI-Hyy_SP12",181,"Evergreen Needleleaf",61.8474006652832,24.2947998046875], + ["Talladega National Forest","TALL_048",135,"Evergreen Needleleaf",32.9504585266113,-87.3932723999023], + ["Talladega National Forest","TALL_049",135,"Evergreen Needleleaf",32.9504585266113,-87.3932723999023], + ["Talladega National Forest","TALL_046",135,"Evergreen Needleleaf",32.9504585266113,-87.3932723999023], + ["Walnut Gulch Kendall","w7",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w8",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w9",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","WalnutGulch",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w20",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w11",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w21",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w13",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w12",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w1",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w15",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w2",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w14",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w3",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w17",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w4",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w16",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w5",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w19",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w6",1531,"Grasslands",31.73643,-109.9418], + ["Walnut Gulch Kendall","w18",1531,"Grasslands",31.73643,-109.9418], + ["Fontainebleau-Barbeau","FR-Fon_CP04",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP10",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP20",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_CP01",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_CP02",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_CP03",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP14",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP03",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP13",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP02",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP12",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP01",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP11",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP18",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP07",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP17",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP06",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP16",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP05",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP15",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP04",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP09",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP19",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Fontainebleau-Barbeau","FR-Fon_SP08",85,"Deciduous Broadleaf Forest",48.476357,2.780096], + ["Bondville","Bondville_IL",230,"Croplands",40.0519218444824,-88.3730926513672], + ["AQUI Fraye-Hillan","A1",49,"Croplands Mosaics",44.4734,-0.7534], + ["AQUI Fraye-Hillan","A2",49,"Croplands Mosaics",44.4734,-0.7534], + ["AQUI Fraye-Hillan","A3",49,"Croplands Mosaics",44.4734,-0.7534], + ["Williams","Williams",1826,"Open Shrublands",35.75503,-112.33744], + ["Daly Uncleared River","DalyUncleared",124,"Woody Savannas",-14.1592,131.3881], + ["Saint Felix de Lauragais","Smos12",337,"Croplands",43.4417,1.88], + ["Savennes","Smos13",158,"Croplands",43.825,1.1767], + ["Mazan-Abbaye","Smos5",1240,"Grassland",44.73383,4.08383], + ["Lajas Experimental Station","LAJA_047",24,"Grasslands",18.02125,-67.0769], + ["Lajas Experimental Station","LAJA_065",24,"Grasslands",18.02125,-67.0769], + ["Lajas Experimental Station","LAJA_054",24,"Grasslands",18.02125,-67.0769], + ["Harvard Forest","HARV_008",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_006",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_026",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_004",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_049",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_027",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_016",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_005",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_013",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_002",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_025",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_014",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_022",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_023",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_012",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_001",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_020",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_021",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_010",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_041",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Harvard Forest","HARV_050",351,"Mixed Forest",42.5377998352051,-72.171501159668], + ["Fyodorovskoye","fy3",251,"Croplands",56.4476,32.9019], + ["Fyodorovskoye","Fyodorovskoye",251,"Croplands",56.4476,32.9019], + ["Fyodorovskoye","fy2",251,"Croplands",56.4476,32.9019], + ["Fyodorovskoye","fy1",251,"Croplands",56.4476,32.9019], + ["Cabrieres Avignon","Smos2",142,"Croplands",43.88367,5.16483], + ["Darabani","Db1",259,"Croplands",48.19477,26.57325], + ["Toravere","Toravere",76,"Cropland Mosaics",58.2657,26.4659888888889], + ["Svartberget","Svartberget_CP03",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP20",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP10",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP11",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP12",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP01",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP13",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP02",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP14",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP03",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP15",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP04",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP16",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP05",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP17",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP06",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP18",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP07",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP19",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP08",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_SP09",242,"Evergreen Needleleaf",64.26,19.77], + ["Svartberget","Svartberget_CP01",242,"Evergreen Needleleaf",64.26,19.77], + ["Underc","UNDE_048",518,"Mixed Forest",46.233959,-89.53751], + ["Underc","UNDE_057",518,"Mixed Forest",46.233959,-89.53751], + ["Underc","UNDE_045",518,"Mixed Forest",46.233959,-89.53751], + ["Creon Armagnac","Smos4",149,"Croplands",43.9936,-0.0469], + ["Salamanca","CasaPeriles",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","LasBrozas",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","LlanosDeLaBoveda",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","Zamarron",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","Canizal",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","LasEritas",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","Guarrati",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","Paredinas",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","LasVacas",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","Carretoro",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","LaCruzDeElias",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","LasTresRayas",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","Granja",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","ConcejoDelMonte",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","ElTomillar",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","LasBodegas",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","LasArenas",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","LasVictorias",779,"Croplands Mosaics",41.2,-5.36], + ["Salamanca","ElCoto",779,"Croplands Mosaics",41.2,-5.36], + ["Fort Peck","Fort_Peck_MT",634,"Grasslands",48.3078308105469,-105.101699829102], + ["Jornada","JORN_055",1329,"Open Shrublands",32.5907,-106.84261], + ["Jornada","JORN_053",1329,"Open Shrublands",32.5907,-106.84261], + ["Jornada","JORN_061",1329,"Open Shrublands",32.5907,-106.84261], + ["Santa Barbara","SantaBarbara",26,"Urban and Built-up",34.4142,-119.8797], + ["Sodankyla","SO1",195,"Evergreen Needleleaf",67.3671,26.6347], + ["Sodankyla","SO3",195,"Evergreen Needleleaf",67.3671,26.6347], + ["Sodankyla","SO2",195,"Evergreen Needleleaf",67.3671,26.6347], + ["Sodankyla","SO4",195,"Evergreen Needleleaf",67.3671,26.6347], + ["Blandy Experimental Farm","BLAN_034",183,"Deciduous Broadleaf",39.0602607727051,-78.0716400146484], + ["Blandy Experimental Farm","BLAN_048",183,"Deciduous Broadleaf",39.0602607727051,-78.0716400146484], + ["Blandy Experimental Farm","BLAN_052",183,"Deciduous Broadleaf",39.0602607727051,-78.0716400146484], + ["Tonzi Ranch","t4",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t5",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t6",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t7",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t8",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t10",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t12",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t11",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t14",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t13",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t16",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t15",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t18",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t17",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t1",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t19",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","TonziRanch",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t2",177,"Woody Savannas",38.430931,-120.965993], + ["Tonzi Ranch","t3",177,"Woody Savannas",38.430931,-120.965993], + ["Smithsonian Environmental Research Center","SERC_004",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_016",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_005",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_017",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_006",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_018",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_007",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_019",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_008",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_009",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_053",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_054",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_021",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_010",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_022",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_011",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_012",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_001",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_013",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_002",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_014",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_003",15,"Croplands",38.89016,-76.5601], + ["Smithsonian Environmental Research Center","SERC_061",15,"Croplands",38.89016,-76.5601], + ["Woodworth","WOOD_027",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_005",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_004",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_069",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_058",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_025",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_003",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_024",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_013",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_002",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_009",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_008",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_029",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_007",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_006",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_023",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_001",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_022",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_011",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_021",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_010",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Woodworth","WOOD_064",579,"Grasslands",47.128231048584,-99.2413635253906], + ["Payerne","Payerne",637,"Evergreen Needleleaf",46.815,6.944] + ] + } +} \ No newline at end of file diff --git a/core/src/main/resources/com/bc/fiduceo/reader/insitu/generic/NDBC_CW_config.json b/core/src/main/resources/com/bc/fiduceo/reader/insitu/generic/NDBC_CW_config.json new file mode 100644 index 000000000..0fda24205 --- /dev/null +++ b/core/src/main/resources/com/bc/fiduceo/reader/insitu/generic/NDBC_CW_config.json @@ -0,0 +1,150 @@ +{ + "name": "NDBC_CW", + "delimiter": "space", + "comment_char": "#", + "regex": "\\w{5}c\\d{4}.txt", + "longitude_name": "longitude", + "latitude_name": "latitude", + "time_vars": ["YY", "MM", "DD", "hh", "mm"], + "location_from_station_database": true, + "variables": [ + { "name": "YY", "type": "int", "long_name": "Year" }, + { "name": "MM", "type": "int", "long_name": "Month" }, + { "name": "DD", "type": "int", "long_name": "Day" }, + { "name": "hh", "type": "int", "long_name": "Hours" }, + { "name": "mm", "type": "int", "long_name": "Minute" }, + { "name": "WDIR", "type": "short", "fill_value": 999, "units": "degT", "long_name": "Ten-minute average wind direction measurements in degrees clockwise from true North.", "cf_standard": "wind_from_direction" }, + { "name": "WSPD", "type": "float", "fill_value": 99.0, "units": "m/s", "long_name": "Ten-minute average wind speed values in m/s.", "cf_standard": "wind_speed" }, + { "name": "GDR", "type": "short", "fill_value": 999, "units": "degT", "long_name": "Direction, in degrees clockwise from true North, of the GST, reported at the last hourly 10-minute segment.", "cf_standard": "wind_gust_from_direction" }, + { "name": "GST", "type": "float", "fill_value": 99.0, "units": "m/s", "long_name": "Peak 5 or 8 second gust speed (m/s) measured during the eight-minute or two-minute period. The 5 or 8 second period can be determined by payload.", "cf_standard": "wind_gust_speed" }, + { "name": "GTIME", "type": "short", "fill_value": 9999, "units": "hhmm", "long_name": "The minute of the hour that the GSP occurred, reported at the last hourly 10-minute segment."} + ], + "station_database": { + "primary_id": "id", + "variables": [ + { "origin": "s","name": "id", "type": "string", "units": "1", "cf_standard": "platform_name", "long_name": "A variable with the standard name of platform_name contains strings which help to identify the platform from which an observation was made. For example, this may be a geographical place name such as \"South Pole\" or the name of a meteorological observing station. A \"platform\" is a structure or vehicle that serves as a base for mounting sensors. Platforms include, but are not limited to, satellites, aeroplanes, ships, buoys, instruments, ground stations, and masts."}, + { "origin": "s","name": "latitude", "type": "float", "units": "degree_north", "cf_standard": "latitude", "long_name": "Latitude is positive northward; its units of degree_north (or equivalent) indicate this explicitly. In a latitude-longitude system defined with respect to a rotated North Pole, the standard name of grid_latitude should be used instead of latitude. Grid latitude is positive in the grid-northward direction, but its units should be plain degree."}, + { "origin": "s","name": "longitude", "type": "float", "units": "degree_east", "cf_standard": "longitude", "long_name": "Longitude is positive eastward; its units of degree_east (or equivalent) indicate this explicitly. In a latitude-longitude system defined with respect to a rotated North Pole, the standard name of grid_longitude should be used instead of longitude. Grid longitude is positive in the grid-eastward direction, but its units should be plain degree."}, + { "origin": "s","name": "station_type", "type": "byte", "fill_value": -1, "units": "1", "long_name": "Station type. 0: OCEAN_BUOY, 1: COAST_BUOY, 2: LAKE_BUOY, 3: OCEAN_STATION, 4: COAST_STATION, 5: LAKE_STATION"}, + { "origin": "s","name": "measurement_type", "type": "byte", "fill_value": -1, "units": "1", "long_name": "Measurement type. 0: CONSTANT_WIND, 1: STANDARD_METEOROLOGICAL"}, + { "origin": "s","name": "anemometer_height", "type": "float", "fill_value": null, "units": "m", "long_name": "Height of instrument above site elevation"} + ], + "stations": [ + ["41002", 31.759, -74.936, 0, 0, 4.1], + ["41004", 32.502, -79.099, 0, 0, 4.1], + ["41008", 31.400, -80.866, 0, 0, 4.9], + ["41009", 28.508, -80.185, 0, 0, 4.1], + ["41010", 28.878, -78.485, 0, 0, 4.1], + ["41025", 35.010, -75.454, 0, 0, 3.8], + ["41040", 14.540, -53.329, 0, 0, 4.1], + ["41041", 14.453, -46.327, 0, 0, 3.8], + ["41043", 21.026, -64.793, 0, 0, 4.1], + ["41044", 21.582, -58.630, 0, 0, 3.8], + ["41046", 23.822, -68.384, 0, 0, 3.8], + ["41047", 27.465, -71.452, 0, 0, 4.1], + ["41048", 31.831, -69.573, 0, 0, 4.1], + ["41049", 27.490, -62.938, 0, 0, 4.1], + ["42001", 25.919, -89.674, 0, 0, 4.1], + ["42002", 26.055, -93.646, 0, 0, 3.8], + ["42003", 25.925, -85.616, 0, 0, 3.2], + ["42012", 25.925, -85.616, 0, 0, 3.8], + ["42019", 27.910, -95.345, 0, 0, 3.2], + ["42020", 26.968, -96.693, 0, 0, 4.1], + ["42035", 29.232, -94.413, 0, 0, 4.1], + ["42036", 28.501, -84.508, 0, 0, 4.1], + ["42039", 28.787, -86.007, 0, 0, 4.1], + ["42040", 29.207, -88.237, 0, 0, 4.1], + ["42055", 22.124, -93.941, 0, 0, 4.1], + ["42056", 19.820, -84.945, 0, 0, 4.1], + ["42057", 16.908, -81.422, 0, 0, 3.8], + ["42058", 14.394, -74.816, 0, 0, 3.8], + ["42059", 15.300, -67.483, 0, 0, 4.1], + ["42060", 16.434, -63.329, 0, 0, 3.8], + ["44005", 43.201, -69.127, 0, 0, 4.9], + ["44007", 43.525, -70.140, 0, 0, 4.1], + ["44008", 40.496, -69.250, 0, 0, 4.1], + ["44009", 38.460, -74.692, 0, 0, 3.8], + ["44011", 41.093, -66.562, 0, 0, 4.1], + ["44013", 42.346, -70.651, 0, 0, 3.2], + ["44014", 36.609, -74.842, 0, 0, 3.2], + ["44017", 40.693, -72.049, 0, 0, 4.1], + ["44018", 42.203, -70.154, 0, 0, 4.1], + ["44020", 41.493, -70.279, 0, 0, 4.1], + ["44025", 40.251, -73.164, 0, 0, 4.1], + ["44027", 44.283, -67.300, 0, 0, 4.1], + ["44065", 40.369, -73.703, 0, 0, 4.1], + ["44066", 39.618, -72.644, 0, 0, 4.1], + ["45001", 48.061, -87.793, 2, 0, 3.2], + ["45002", 45.344, -86.411, 2, 0, 3.2], + ["45003", 45.351, -82.840, 2, 0, 5.0], + ["45004", 47.585, -86.585, 2, 0, 3.6], + ["45006", 47.335, -89.793, 2, 0, 3.6], + ["45007", 42.674, -87.026, 2, 0, 3.6], + ["45008", 44.283, -82.416, 2, 0, 3.6], + ["45012", 43.621, -77.401, 2, 0, 3.6], + ["46001", 56.300, -148.018, 0, 0, 3.8], + ["46002", 42.662, -130.507, 0, 0, 3.8], + ["46005", 46.134, -131.079, 0, 0, 4.1], + ["46011", 34.936, -120.998, 0, 0, 3.8], + ["46012", 37.356, -122.881, 0, 0, 3.8], + ["46013", 38.235, -123.317, 0, 0, 3.8], + ["46015", 42.752, -124.844, 0, 0, 4.1], + ["46022", 40.748, -124.527, 0, 0, 3.8], + ["46025", 33.755, -119.045, 0, 0, 3.8], + ["46026", 37.754, -122.839, 0, 0, 3.8], + ["46027", 41.840, -124.382, 0, 0, 3.8], + ["46028", 35.770, -121.903, 0, 0, 4.1], + ["46029", 46.163, -124.487, 0, 0, 3.8], + ["46035", 57.016, -177.703, 0, 0, 5.0], + ["46041", 47.352, -124.739, 0, 0, 3.8], + ["46042", 36.785, -122.396, 0, 0, 3.8], + ["46047", 32.388, -119.525, 0, 0, 3.8], + ["46050", 44.669, -124.546, 0, 0, 4.1], + ["46054", 34.274, -120.468, 0, 0, 3.8], + ["46060", 60.586, -146.787, 0, 0, 4.9], + ["46061", 60.238, -146.833, 0, 0, 4.9], + ["46066", 52.765, -155.009, 0, 0, 3.8], + ["46069", 33.677, -120.213, 0, 0, 3.8], + ["46070", 55.008, 175.183, 0, 0, 5.0], + ["46071", 51.022, 179.784, 0, 0, 3.8], + ["46072", 51.666, -172.114, 0, 0, 3.8], + ["46073", 55.008, -172.012, 0, 0, 4.9], + ["46075", 53.969, -160.794, 0, 0, 3.8], + ["46076", 59.471, -148.009, 0, 0, 3.8], + ["46077", 57.869, -154.211, 0, 0, 4.9], + ["46078", 55.557, -152.641, 0, 0, 3.8], + ["46080", 57.947, -150.042, 0, 0, 4.9], + ["46081", 60.802, -148.283, 1, 0, 4.9], + ["46082", 59.670, -143.353, 0, 0, 5.0], + ["46083", 58.270, -138.019, 0, 0, 4.9], + ["46084", 56.622, -136.102, 0, 0, 3.8], + ["46085", 55.878, -142.876, 0, 0, 3.8], + ["46086", 32.499, -118.052, 0, 0, 4.1], + ["46087", 48.493, -124.727, 0, 0, 3.8], + ["46088", 48.332, -123.179, 0, 0, 3.8], + ["46089", 45.936, -125.793, 0, 0, 3.8], + ["51000", 23.528, -153.792, 0, 0, 4.1], + ["51002", 17.042, -157.746, 0, 0, 4.1], + ["51003", 19.196, -160.639, 0, 0, 3.8], + ["51004", 17.538, -152.230, 0, 0, 4.1], + ["51101", 24.359, -162.081, 0, 0, 4.1], + ["BLIA2", 60.839, -146.884, 4, 0, 21.6], + ["BURL1", 28.905, -89.428, 4, 0, 38.0], + ["BUZM3", 41.397, -71.033, 3, 0, 24.8], + ["CDRF1", 29.136, -83.029, 4, 0, 10.0], + ["CHLV2", 36.905, -75.713, 3, 0, 43.3], + ["FWYF1", 25.591, -80.097, 3, 0, 43.9], + ["IOSN3", 42.967, -70.623, 4, 0, 19.2], + ["LONF1", 24.844, -80.864, 4, 0, 6.3], + ["MDRM1", 43.969, -68.128, 3, 0, 22.6], + ["MISM1", 43.784, -68.855, 4, 0, 7.1], + ["MLRF1", 25.012, -80.376, 3, 0, 15.8], + ["PILA2", 59.742, -149.470, 4, 0, 6.6], + ["PLSF1", 24.693, -82.773, 3, 0, 17.7], + ["SGOF1", 29.408, -84.858, 3, 0, 35.1], + ["SISW1", 48.321, -122.831, 3, 0, 20.2], + ["SPGF1", 26.704, -78.995, 4, 0, 6.6], + ["STDM4", 47.184, -87.225, 5, 0, 35.2] + ] + } +} \ No newline at end of file diff --git a/core/src/main/resources/com/bc/fiduceo/reader/insitu/generic/NDBC_SM_config.json b/core/src/main/resources/com/bc/fiduceo/reader/insitu/generic/NDBC_SM_config.json new file mode 100644 index 000000000..ef1a7c5fe --- /dev/null +++ b/core/src/main/resources/com/bc/fiduceo/reader/insitu/generic/NDBC_SM_config.json @@ -0,0 +1,490 @@ +{ + "name": "NDBC_SM", + "delimiter": "space", + "comment_char": "#", + "regex": "\\w{5}h\\d{4}.txt", + "longitude_name": "longitude", + "latitude_name": "latitude", + "time_vars": ["YY", "MM", "DD", "hh", "mm"], + "location_from_station_database": true, + "variables": [ + { "name": "YY", "type": "int", "long_name": "Year" }, + { "name": "MM", "type": "int", "long_name": "Month" }, + { "name": "DD", "type": "int", "long_name": "Day" }, + { "name": "hh", "type": "int", "long_name": "Hours" }, + { "name": "mm", "type": "int", "long_name": "Minute" }, + { "name": "WDIR", "type": "short", "fill_value": 999, "units": "degT", "long_name": "Ten-minute average wind direction measurements in degrees clockwise from true North.", "cf_standard": "wind_from_direction" }, + { "name": "WSPD", "type": "float", "fill_value": 99.0, "units": "m/s", "long_name": "Ten-minute average wind speed values in m/s.", "cf_standard": "wind_speed" }, + { "name": "GST", "type": "float", "fill_value": 99.0, "units": "m/s", "long_name": "Peak 5 or 8 second gust speed (m/s) measured during the eight-minute or two-minute period. The 5 or 8 second period can be determined by payload.", "cf_standard": "wind_gust_speed" }, + { "name": "WVHT", "type": "float", "fill_value": 99.0, "units": "m", "long_name": "Significant wave height (meters) is calculated as the average of the highest one-third of all of the wave heights during the 20-minute sampling period.", "cf_standard": "sea_surface_wave_significant_height" }, + { "name": "DPD", "type": "float", "fill_value": 99.0, "units": "s", "long_name": "Dominant wave period (seconds) is the period with the maximum wave energy."}, + { "name": "APD", "type": "float", "fill_value": 99.0, "units": "s", "long_name": "Average wave period (seconds) of all waves during the 20-minute period.", "cf_standard": "sea_surface_wave_mean_period" }, + { "name": "MWD", "type": "short", "fill_value": 999, "units": "degT", "long_name": "The direction from which the waves at the dominant period (DPD) are coming. The units are degrees from true North, increasing clockwise, with North as 0 (zero) degrees and East as 90 degrees."}, + { "name": "PRES", "type": "float", "fill_value": 99.0, "units": "hPa", "long_name": "Sea level pressure (hPa).", "cf_standard": "air_pressure_at_mean_sea_level" }, + { "name": "ATMP", "type": "float", "fill_value": 99.0, "units": "degC", "long_name": "Air temperature (Celsius). For sensor heights see variable 'air_temp_height'.", "cf_standard": "air_temperature" }, + { "name": "WTMP", "type": "float", "fill_value": 999.0, "units": "degC", "long_name": "Sea surface temperature (Celsius). For buoys the depth is referenced to the hull's waterline. For fixed platforms it varies with tide, but is referenced to, or near Mean Lower Low Water (MLLW).", "cf_standard": "sea_surface_temperature" }, + { "name": "DEWP", "type": "float", "fill_value": 99.0, "units": "degC", "long_name": "Dewpoint temperature taken at the same height as the air temperature measurement.", "cf_standard": "dew_point_temperature" }, + { "name": "VIS", "type": "float", "fill_value": 99.0, "units": "mi", "long_name": "Station visibility (nautical miles). Note that buoy stations are limited to reports from 0 to 1.6 nmi.", "cf_standard": "visibility_in_air" }, + { "name": "TIDE", "type": "float", "fill_value": 99.0, "units": "ft", "long_name": "The water level in feet above or below Mean Lower Low Water (MLLW).", "cf_standard": "" } + ], + "station_database": { + "primary_id": "id", + "variables": [ + { "origin": "s","name": "id", "type": "string", "units": "1", "cf_standard": "platform_name", "long_name": "Station identifier"}, + { "origin": "s","name": "latitude", "type": "float", "units": "degree_north", "cf_standard": "latitude", "long_name": "Latitude is positive northward; its units of degree_north (or equivalent) indicate this explicitly. In a latitude-longitude system defined with respect to a rotated North Pole, the standard name of grid_latitude should be used instead of latitude. Grid latitude is positive in the grid-northward direction, but its units should be plain degree."}, + { "origin": "s","name": "longitude", "type": "float", "units": "degree_east", "cf_standard": "longitude", "long_name": "Longitude is positive eastward; its units of degree_east (or equivalent) indicate this explicitly. In a latitude-longitude system defined with respect to a rotated North Pole, the standard name of grid_longitude should be used instead of longitude. Grid longitude is positive in the grid-eastward direction, but its units should be plain degree."}, + { "origin": "s","name": "station_type", "type": "byte", "fill_value": -1, "units": "1", "long_name": "Station type. 0: OCEAN_BUOY, 1: COAST_BUOY, 2: LAKE_BUOY, 3: OCEAN_STATION, 4: COAST_STATION, 5: LAKE_STATION"}, + { "origin": "s","name": "measurement_type", "type": "byte", "fill_value": -1, "units": "1", "long_name": "Measurement type. 0: CONSTANT_WIND, 1: STANDARD_METEOROLOGICAL"}, + { "origin": "s","name": "anemometer_height", "type": "float", "fill_value": null, "units": "m", "long_name": "Height of instrument above site elevation"}, + { "origin": "s","name": "air_temp_height", "type": "float", "fill_value": null, "units": "m", "long_name": "Height of instrument above site elevation"}, + { "origin": "s","name": "barometer_height", "type": "float", "fill_value": null, "units": "m", "long_name": "Height of instrument above above mean sea level"}, + { "origin": "s","name": "sst_depth", "type": "float", "fill_value": null, "units": "m", "long_name": "Depth of instrument below water line"} + ], + "stations": [ + ["34002", -55.000, -90.000, 0, 1, 6.20, 5.70, 5.60, 0.80], + ["41001", 34.714, -72.248, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["41002", 31.759, -74.936, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["41004", 32.502, -79.099, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["41008", 31.400, -80.866, 0, 1, 4.90, 4.40, 0.30, 0.60], + ["41009", 28.508, -80.185, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["41013", 33.441, -77.764, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["41024", 33.837, -78.477, 1, 1, 2.95, 2.79, 2.79, 1.00], + ["41025", 35.010, -75.454, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["41029", 32.803, -79.624, 0, 1, 2.95, 2.79, 2.79, 1.00], + ["41033", 32.279, -80.406, 0, 1, 2.95, 2.79, 2.79, 1.00], + ["41037", 33.988, -77.362, 0, 1, 2.95, 2.79, 2.79, 1.00], + ["41038", 34.141, -77.715, 0, 1, 2.95, 2.79, 2.79, 1.00], + ["41040", 14.540, -53.329, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["41041", 14.453, -46.327, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["41043", 21.026, -64.793, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["41044", 21.582, -58.630, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["41046", 23.822, -68.384, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["41047", 27.465, -71.452, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["41048", 31.831, -69.573, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["41049", 27.490, -62.938, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["41051", 18.257, -65.004, 0, 1, 4.00, 4.00, 2.00, 1.00], + ["41052", 18.249, -64.763, 0, 1, 4.00, 3.00, 3.00, 1.00], + ["41053", 18.474, -66.099, 1, 1, 4.00, 3.00, 3.00, 1.00], + ["41056", 18.261, -65.464, 0, 1, 4.00, 3.00, 3.00, 1.00], + ["41057", 19.833, -70.731, 1, 1, 3.35, null, null, null], + ["41058", 18.476, -65.157, 0, 1, 4.00, 4.00, 2.00, 1.00], + ["41062", 35.778, -75.095, 0, 1, 3.50, 1.50, 1.50, 2.00], + ["41063", 34.782, -75.941, 0, 1, 3.50, 1.50, 1.50, 2.00], + ["41064", 34.207, -76.949, 0, 1, 2.95, 2.79, 2.79, null], + ["42001", 25.919, -89.674, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["42002", 26.055, -93.646, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["42003", 25.925, -85.616, 0, 1, 3.20, 2.80, 1.80, 1.10], + ["42012", 30.060, -87.548, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["42013", 27.173, -82.924, 0, 1, 3.10, 2.00, 1.90, 1.00], + ["42019", 27.910, -95.345, 0, 1, 3.20, 2.80, 1.80, 1.10], + ["42022", 27.505, -83.741, 0, 1, 3.10, 2.00, 1.90, 1.00], + ["42035", 29.232, -94.413, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["42036", 28.501, -84.508, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["42039", 28.787, -86.007, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["42040", 29.207, -88.237, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["42043", 28.982, -94.899, 0, 1, 4.00, 4.00, 4.00, 2.00], + ["42044", 26.191, -97.051, 0, 1, 4.00, 4.00, 4.00, 2.00], + ["42045", 26.217, -96.500, 0, 1, 4.00, 4.00, 4.00, 2.00], + ["42046", 27.890, -94.037, 0, 1, 4.00, 4.00, 4.00, 2.00], + ["42047", 27.897, -93.597, 0, 1, 4.00, 4.00, 4.00, 2.00], + ["42055", 22.124, -93.941, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["42056", 19.820, -84.945, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["42057", 16.908, -81.422, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["42058", 14.394, -74.816, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["42059", 15.300, -67.483, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["42060", 16.434, -63.329, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["42067", 30.050, -88.583, 0, 1, 4.00, 4.00, 4.00, null], + ["42085", 17.870, -66.537, 1, 1, 4.00, 3.00, 3.00, 1.00], + ["42087", 11.185, -60.848, 1, 1, 3.35, null, null, null], + ["42088", 11.301, -60.521, 1, 1, 3.35, null, null, null], + ["42089", 19.699, -80.061, 1, 1, 3.35, 3.00, 3.00, 1.00], + ["42090", 18.432, -69.580, 1, 1, 3.35, null, null, null], + ["44005", 43.201, -69.127, 0, 1, 4.90, 4.40, 0.30, 0.60], + ["44007", 43.525, -70.140, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["44008", 40.496, -69.250, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["44009", 38.460, -74.692, 0, 1, 3.80, 3.40, 3.40, 2.00], + ["44011", 41.093, -66.562, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["44013", 42.346, -70.651, 0, 1, 3.20, 2.80, 1.80, 1.10], + ["44014", 36.609, -74.842, 0, 1, 3.20, 2.80, 1.80, 1.10], + ["44017", 40.693, -72.049, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["44018", 42.203, -70.154, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["44020", 41.493, -70.279, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["44022", 40.883, -73.728, 1, 1, 3.50, 3.00, null, 1.00], + ["44024", 42.325, -65.909, 0, 1, 4.00, 3.00, 3.00, 1.00], + ["44025", 40.251, -73.164, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["44027", 44.283, -67.300, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["44029", 42.523, -70.566, 0, 1, 4.00, 3.00, 3.00, 1.00], + ["44030", 43.179, -70.426, 0, 1, 4.00, 3.00, 3.00, 1.00], + ["44032", 43.715, -69.355, 0, 1, 4.00, 3.00, 3.00, 1.00], + ["44033", 44.055, -68.996, 1, 1, 4.00, 3.00, 3.00, 1.00], + ["44034", 44.103, -68.112, 0, 1, 4.00, 3.00, 3.00, 1.00], + ["44037", 43.497, -67.876, 0, 1, 4.00, 3.00, 3.00, 1.00], + ["44039", 41.138, -72.655, 0, 1, 3.50, 3.00, null, 1.00], + ["44040", 40.956, -73.580, 0, 1, 3.50, 3.00, null, 1.00], + ["44041", 37.211, -76.787, 1, 1, 3.00, 3.00, 1.00, 0.50], + ["44042", 38.033, -76.335, 1, 1, 3.00, 3.00, 1.00, 0.50], + ["44043", 39.152, -76.391, 1, 1, 3.00, 3.00, 1.00, 0.50], + ["44057", 39.540, -76.074, 1, 1, 3.00, 3.00, 1.00, 0.50], + ["44058", 37.567, -76.257, 1, 1, 3.00, 3.00, 1.00, 0.50], + ["44059", 36.846, -76.298, 1, 1, 3.00, 3.00, 1.00, 0.50], + ["44060", 41.263, -72.067, 1, 1, 3.50, 3.00, null, 1.00], + ["44062", 38.556, -76.415, 0, 1, 3.00, 3.00, 1.00, 0.50], + ["44063", 38.963, -76.448, 1, 1, 3.00, 3.00, 1.00, 0.50], + ["44064", 36.998, -76.087, 0, 1, 3.00, 3.00, 1.00, 0.50], + ["44065", 40.369, -73.703, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["44066", 39.618, -72.644, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["44069", 40.698, -73.087, 1, 1, 3.00, 3.00, null, 1.00], + ["44072", 37.201, -76.266, 0, 1, 3.00, 3.00, 1.00, 0.50], + ["45001", 48.061, -87.793, 2, 1, 3.20, 2.80, 184.80, 1.10], + ["45002", 45.344, -86.411, 2, 1, 3.20, 2.80, 178.40, 1.10], + ["45003", 45.351, -82.840, 2, 1, 5.00, 4.50, 177.50, 0.40], + ["45004", 47.585, -86.585, 2, 1, 3.60, 3.30, 185.30, 1.30], + ["45005", 41.677, -82.398, 2, 1, 3.50, 3.80, 176.30, 1.60], + ["45006", 47.335, -89.793, 2, 1, 3.60, 3.30, 185.30, 1.30], + ["45007", 42.674, -87.026, 2, 1, 3.60, 3.30, 178.30, 1.30], + ["45008", 44.283, -82.416, 2, 1, 3.60, 3.30, 179.30, 1.30], + ["45012", 43.621, -77.401, 2, 1, 3.60, 3.30, 77.00, 1.30], + ["45013", 43.100, -87.850, 2, 1, 2.00, 2.00, 178.00, 0.00], + ["45014", 44.795, -87.759, 2, 1, 2.00, 2.00, 178.00, 0.00], + ["45020", 44.789, -85.604, 2, 1, 2.40, 2.40, null, null], + ["45022", 45.405, -85.087, 2, 1, 2.95, 2.64, 178.64, 1.00], + ["45023", 47.270, -88.607, 2, 1, 3.00, 2.80, null, 3.00], + ["45024", 43.981, -86.556, 2, 1, 2.95, 2.64, 178.64, null], + ["45025", 46.969, -88.398, 2, 1, 3.00, 2.80, 185.80, 3.00], + ["45026", 41.982, -86.619, 2, 1, 3.00, 3.00, null, 1.00], + ["45027", 46.860, -91.930, 2, 1, 3.00, 3.00, 185.50, 1.00], + ["45028", 46.814, -91.829, 2, 1, 3.00, 3.00, 185.50, 1.00], + ["45029", 42.900, -86.272, 2, 1, 3.00, 3.00, null, 1.00], + ["45161", 43.182, -86.360, 2, 1, 2.30, 2.30, null, 1.00], + ["45162", 44.988, -83.270, 2, 1, 2.30, 2.30, null, 1.00], + ["45163", 43.985, -83.596, 2, 1, 2.30, 2.30, null, 1.00], + ["45164", 41.748, -81.698, 2, 1, 1.00, 2.30, null, 1.00], + ["45165", 41.702, -83.261, 2, 1, 2.00, 2.00, null, 1.00], + ["45166", 44.785, -73.258, 2, 1, 2.50, 2.00, null, 1.00], + ["45167", 42.185, -80.135, 2, 1, 2.00, 2.00, 175.00, 1.00], + ["45168", 42.397, -86.331, 2, 1, 3.00, 3.00, null, 1.00], + ["45169", 41.615, -81.821, 2, 1, 2.75, 2.75, 176.75, 0.70], + ["45169", 41.615, -81.821, 2, 1, 2.75, 2.75, 176.75, 0.70], + ["45170", 41.755, -86.968, 2, 1, 1.50, 1.00, null, 1.00], + ["45171", 46.724, -87.411, 2, 1, 0.82, 0.82, 178.82, 0.70], + ["45172", 46.740, -85.980, 2, 1, 0.82, 0.82, 178.82, 0.70], + ["45173", 46.573, -86.572, 2, 1, 2.00, 2.75, 180.75, 1.00], + ["45174", 42.135, -87.655, 2, 1, 3.00, 3.00, 179.00, 1.00], + ["45175", 45.825, -84.772, 2, 1, 3.00, 3.00, 179.00, 1.00], + ["45176", 41.550, -81.765, 2, 1, 3.00, 3.00, 177.00, 1.00], + ["45178", 44.603, -73.394, 2, 1, 1.00, 1.00, 31.00, 1.00], + ["45179", 47.195, -87.224, 2, 1, 0.60, 0.60, 178.60, 0.70], + ["45183", 44.982, -85.831, 2, 1, 2.00, 2.00, 178.00, 1.00], + ["45186", 42.368, -87.795, 2, 1, 1.50, 1.00, null, 1.00], + ["45187", 42.491, -87.779, 2, 1, 1.50, 1.00, null, 1.00], + ["45T01", 48.008, -87.666, 2, 1, 3.20, 2.80, 185.00, 0.50], + ["46001", 56.300, -148.018, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46002", 42.662, -130.507, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46005", 46.134, -131.079, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["46006", 40.764, -137.377, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["46011", 34.936, -120.998, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46012", 37.356, -122.881, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46013", 38.235, -123.317, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46014", 39.231, -123.974, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46015", 42.752, -124.844, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["46022", 40.748, -124.527, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46025", 33.755, -119.045, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46026", 37.754, -122.839, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46027", 41.840, -124.382, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46028", 35.770, -121.903, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["46029", 46.163, -124.487, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46035", 57.016, -177.703, 0, 1, 5.00, 4.50, 0.50, 0.40], + ["46041", 47.352, -124.739, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46042", 36.785, -122.396, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46047", 32.388, -119.525, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46050", 44.669, -124.546, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["46053", 34.241, -119.839, 0, 1, 3.20, 2.80, 1.80, 1.10], + ["46054", 34.274, -120.468, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46059", 38.053, -129.967, 0, 1, 3.70, 3.70, 3.70, 1.50], + ["46060", 60.586, -146.787, 0, 1, 4.90, 4.40, 0.30, 0.60], + ["46061", 60.238, -146.833, 0, 1, 4.90, 4.40, 0.30, 0.60], + ["46066", 52.765, -155.009, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46069", 33.677, -120.213, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46070", 55.008, 175.183, 0, 1, 5.00, 4.50, 0.50, 0.40], + ["46071", 51.022, 179.784, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46072", 51.666, -172.114, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46073", 55.008, -172.012, 0, 1, 4.90, 4.40, 0.30, 0.60], + ["46075", 53.969, -160.794, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46076", 59.471, -148.009, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46077", 57.869, -154.211, 0, 1, 4.90, 4.40, 0.30, 0.60], + ["46078", 55.557, -152.641, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46080", 57.947, -150.042, 0, 1, 4.90, 4.40, 0.30, 0.60], + ["46081", 60.802, -148.283, 1, 1, 4.90, 4.40, 0.30, 0.60], + ["46082", 59.670, -143.353, 0, 1, 5.00, 4.50, 0.50, 0.40], + ["46083", 58.270, -138.019, 0, 1, 4.90, 4.40, 0.30, 0.60], + ["46084", 56.622, -136.102, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46085", 55.878, -142.876, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46086", 32.499, -118.052, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["46087", 48.493, -124.727, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46088", 48.332, -123.179, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46089", 45.936, -125.793, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["46094", 44.642, -124.300, 0, 1, 3.00, 3.00, 3.00, 1.70], + ["46096", 46.173, -124.127, 0, 1, 3.00, 3.00, 3.00, 1.00], + ["46097", 44.639, -124.304, 0, 1, 4.50, 4.00, 3.95, 1.15], + ["46098", 44.378, -124.947, 0, 1, 4.50, 4.00, 3.95, 1.15], + ["46099", 46.988, -124.567, 0, 1, 4.50, 4.00, 3.95, 1.15], + ["46100", 46.851, -124.964, 0, 1, 4.50, 4.00, 3.95, 1.15], + ["46118", 48.724, -122.576, 1, 1, 2.97, 2.00, 2.00, 0.50], + ["46119", 47.967, -124.950, 0, 1, 3.70, 3.70, null, 1.00], + ["46125", 47.907, -122.627, 1, 1, 2.10, 2.10, 1.00, 3.00], + ["46127", 46.215, -124.063, 1, 1, 3.00, 3.00, 3.00, 0.00], + ["51000", 23.528, -153.792, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["51001", 24.451, -162.008, 0, 1, 3.20, 2.80, 1.80, 1.10], + ["51002", 17.042, -157.746, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["51003", 19.196, -160.639, 0, 1, 3.80, 3.40, 2.40, 2.00], + ["51004", 17.538, -152.230, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["51101", 24.359, -162.081, 0, 1, 4.10, 3.70, 2.70, 1.50], + ["AAMC1", 37.772, -122.300, 4, 1, 6.90, 4.30, 5.30, 1.20], + ["ADKA2", 51.861, -176.637, 4, 1, 7.00, 6.30, 7.00, 2.80], + ["AGMW3", 44.608, -87.433, 5, 1, 9.00, 9.00, null, null], + ["AGXC1", 33.716, -118.246, 4, 1, 20.00, null, null, null], + ["AJXA2", 58.287, -134.398, 4, 1, 3.00, 3.00, 7.60, null], + ["ALIA2", 56.898, -154.247, 4, 1, 10.30, 9.70, 6.50, 2.20], + ["AMAA2", 58.915, -151.952, 4, 1, 15.70, 15.70, 35.10, null], + ["AMRL1", 29.450, -91.338, 4, 1, 11.00, 10.50, 8.50, 1.90], + ["ANPT2", 27.837, -97.039, 4, 1, 3.96, 3.14, 11.78, 10.81], + ["ANTA2", 61.238, -149.890, 4, 1, 6.90, 6.30, 9.00, 1.92], + ["ANVC1", 38.915, -123.711, 4, 1, 7.60, 7.00, 9.40, 1.70], + ["APCF1", 29.724, -84.980, 4, 1, 7.00, 5.40, 6.20, 2.00], + ["APNM4", 45.060, -83.424, 4, 1, 18.00, 15.00, null, null], + ["APRP7", 13.444, 144.657, 4, 1, 8.40, 9.70, 5.10, 2.00], + ["AROP4", 18.480, -66.702, 4, 1, 6.40, 3.40, null, null], + ["ARPF1", 28.433, -82.667, 4, 1, 11.20, 6.90, 6.70, 0.10], + ["ASTO3", 46.207, -123.768, 4, 1, 6.90, 6.40, 5.00, 1.70], + ["ATGM1", 44.392, -68.204, 4, 1, 8.60, 7.90, 5.00, 1.20], + ["ATKA2", 52.232, -174.173, 4, 1, 8.50, 7.90, 6.20, 2.20], + ["AUGA2", 59.378, -153.348, 4, 1, 6.60, 6.40, 10.60, null], + ["AWRT2", 28.227, -96.796, 4, 1, null, null, null, 8.87], + ["BABT2", 27.297, -97.405, 4, 1, 10.00, null, 4.00, null], + ["BARA9", 17.591, -61.821, 4, 1, 9.70, 7.70, 7.50, null], + ["BAXC1", 33.766, -118.240, 4, 1, 62.40, null, null, null], + ["BEPB6", 32.374, -64.701, 4, 1, 4.60, 4.30, 5.10, 2.80], + ["BFTN7", 34.717, -76.671, 4, 1, 7.00, 3.80, 3.00, 4.40], + ["BGCF1", 26.404, -81.881, 4, 1, 17.30, 16.60, 16.60, null], + ["BHRI3", 41.646, -87.147, 5, 1, 10.00, 2.40, 181.90, null], + ["BIGM4", 46.827, -87.727, 5, 1, 10.00, 3.90, 186.80, null], + ["BISM2", 38.220, -76.039, 4, 1, 7.10, 3.10, 2.80, 10.80], + ["BLIA2", 60.839, -146.884, 4, 1, 21.60, 21.30, 16.50, null], + ["BLTA2", 58.455, -135.889, 4, 1, 9.10, 5.50, 4.80, null], + ["BRHC3", 41.174, -73.181, 4, 1, 7.60, 3.20, 4.00, 4.90], + ["BRND1", 38.987, -75.113, 3, 1, 21.10, 20.60, 8.30, 0.46], + ["BSBM4", 44.055, -86.514, 5, 1, 10.00, 2.70, 185.90, null], + ["BSCA1", 30.329, -87.829, 4, 1, 7.01, 6.40, 2.85, 2.00], + ["BUFN6", 42.878, -78.890, 5, 1, 8.50, 6.50, 179.00, 4.50], + ["BUZM3", 41.397, -71.033, 3, 1, 24.80, 24.50, 17.40, null], + ["BZST2", 26.067, -97.155, 4, 1, 4.25, 3.23, 11.82, 10.60], + ["CAPL1", 29.768, -93.343, 4, 1, 12.30, 11.70, 10.50, 0.60], + ["CBBV2", 36.967, -76.114, 3, 1, 6.10, 4.20, 5.80, 3.20], + ["CBLO1", 41.981, -80.556, 5, 1, 12.00, 9.10, 189.60, null], + ["CBRW3", 45.198, -87.360, 5, 1, 10.00, 3.30, 181.30, null], + ["CDEA2", 56.001, -134.136, 4, 1, 10.00, 1.50, 16.50, null], + ["CDRF1", 29.136, -83.029, 4, 1, 10.00, 9.10, 3.00, null], + ["CDXA2", 56.001, -134.136, 4, 1, 9.10, 9.10, 24.40, null], + ["CECC1", 41.746, -124.184, 4, 1, 8.60, 7.90, 4.80, 1.30], + ["CFWM1", 44.657, -67.205, 4, 1, 8.50, 7.90, 5.30, 0.20], + ["CHAO3", 43.351, -124.337, 4, 1, 8.40, 3.00, 4.50, 2.00], + ["CHAV3", 18.335, -64.920, 4, 1, 6.50, 2.60, 2.30, 0.90], + ["CHBV2", 37.032, -76.083, 3, 1, 4.60, 3.10, null, 3.30], + ["CHDS1", 33.661, -82.199, 5, 1, 2.10, 0.60, 116.40, null], + ["CHII2", 41.916, -87.572, 5, 1, 25.90, 22.90, null, null], + ["CHLV2", 36.905, -75.713, 3, 1, 43.30, 22.30, 23.40, null], + ["CHSV3", 17.748, -64.699, 4, 1, 7.10, 2.80, 3.00, 1.20], + ["CHYV2", 36.926, -76.007, 4, 1, 27.90, null, null, null], + ["CHYW1", 48.863, -122.759, 4, 1, 7.00, 3.70, 8.20, 3.10], + ["CLBF1", 27.736, -82.686, 4, 1, 11.00, 10.30, 10.20, 1.00], + ["CLKN7", 34.622, -76.525, 4, 1, 9.80, 6.40, 11.00, null], + ["CLSM4", 42.471, -82.877, 5, 1, 10.00, 3.30, 176.80, null], + ["CMTI2", 41.730, -87.538, 5, 1, 10.30, 5.70, 179.80, null], + ["CNDO1", 41.542, -81.637, 5, 1, 7.80, 3.90, 177.70, 4.20], + ["CNII2", 41.856, -87.609, 5, 1, 10.00, 1.80, null, null], + ["CPMW1", 48.860, -122.756, 4, 1, 7.00, 3.30, null, null], + ["CPXC1", 35.170, -120.741, 4, 1, 4.00, 4.00, 4.00, null], + ["CRTA1", 30.308, -88.140, 4, 1, 12.23, 6.45, 4.01, 2.30], + ["CRVA2", 60.558, -145.752, 4, 1, 8.40, 7.80, 6.60, 3.00], + ["CRYV2", 36.888, -76.338, 4, 1, 8.60, 2.80, 3.00, null], + ["CSPA2", 58.199, -136.640, 4, 1, 10.00, 1.50, 26.50, null], + ["CWBF1", 27.978, -82.832, 4, 1, 6.70, 3.20, 4.70, 3.70], + ["DBLN6", 42.494, -79.354, 5, 1, 20.40, 12.80, 196.90, null], + ["DESW1", 47.675, -124.485, 4, 1, 31.40, 31.40, 39.00, null], + ["DISW3", 47.079, -90.728, 5, 1, 25.30, 25.30, 210.60, null], + ["DKCM6", 30.356, -88.567, 4, 1, 8.40, 3.60, 210.60, 1.50], + ["DPIA1", 30.250, -88.075, 4, 1, 13.50, 9.10, 8.50, null], + ["DPXC1", 38.056, -122.264, 4, 1, 7.60, 3.40, null, null], + ["DRFA2", 60.553, -152.137, 4, 1, 6.60, 6.40, 17.40, null], + ["DTLM4", 45.993, -83.898, 5, 1, 11.70, 6.40, 180.20, 4.20], + ["DUKN7", 36.184, -75.746, 4, 1, 8.60, 8.00, 9.90, 9.20], + ["DULM5", 46.776, -92.092, 5, 1, 13.20, 7.10, 185.80, 4.20], + ["EINL1", 29.373, -91.384, 4, 1, 4.00, 1.50, 1.50, null], + ["EMAT2", 28.710, -95.914, 4, 1, 8.20, 5.90, 5.70, 2.10], + ["EPTT2", 29.481, -94.917, 4, 1, 5.70, 4.30, 6.00, 1.30], + ["EREP1", 42.172, -80.098, 5, 1, 10.70, null, null, null], + ["ESPP4", 18.094, -65.471, 4, 1, 8.50, 8.20, 3.40, 1.60], + ["FAIO1", 41.764, -81.281, 5, 1, 20.90, 3.20, 178.00, 4.40], + ["FBIS1", 32.685, -79.888, 4, 1, 9.80, 6.70, 4.60, null], + ["FHPF1", 28.153, -82.801, 4, 1, 9.40, 5.40, 5.30, 0.60], + ["FILA2", 59.332, -151.995, 4, 1, 15.70, 15.70, 19.90, null], + ["FMOA1", 30.228, -88.024, 4, 1, 36.20, 17.50, 31.00, null], + ["FMRF1", 26.647, -81.871, 4, 1, 6.90, 3.30, 0.60, 1.00], + ["FPST2", 28.936, -95.294, 4, 1, 15.00, 13.00, 13.00, null], + ["FRDP4", 18.335, -65.631, 4, 1, 6.40, 3.40, 1.50, 9.85], + ["FTGM4", 43.007, -82.422, 5, 1, 27.20, 3.60, 180.50, null], + ["FTPC1", 37.806, -122.466, 4, 1, 7.30, 6.30, 4.70, 1.20], + ["FWYF1", 25.591, -80.097, 3, 1, 43.90, 11.00, 29.30, 1.00], + ["GELO1", 41.859, -80.975, 5, 1, 8.00, 7.30, 185.90, null], + ["GISL1", 29.265, -89.958, 4, 1, 6.60, 3.90, 4.20, 0.80], + ["GIXA2", 55.446, -131.881, 4, 1, 2.40, 2.40, 11.20, null], + ["GNJT2", 29.357, -94.725, 4, 1, 12.10, 10.90, 7.10, 1.80], + ["GRBL1", 29.101, -89.978, 3, 1, 10.00, 10.00, 3.00, 1.00], + ["GRIM4", 46.721, -87.412, 5, 1, 11.00, 11.50, 209.50, null], + ["GRMM4", 46.684, -85.972, 5, 1, 9.00, 7.90, 191.40, null], + ["GTLM4", 45.211, -85.550, 5, 1, 16.00, 1.50, 183.80, null], + ["GTRM4", 47.179, -88.241, 5, 1, 10.00, 3.30, 191.10, null], + ["HBYC1", 40.767, -124.217, 4, 1, 13.60, 13.20, 7.80, 1.20], + ["HCGN7", 35.209, -75.704, 4, 1, 7.60, 5.90, 6.40, 1.40], + ["HHLO1", 41.401, -82.545, 5, 1, 10.00, 7.00, 183.50, null], + ["HLNM4", 42.773, -86.213, 5, 1, 10.40, 6.30, 180.40, 3.90], + ["HMSA2", 59.602, -151.417, 4, 1, 4.00, null, null, null], + ["HRBM4", 43.846, -82.643, 5, 1, 8.30, 5.70, 179.60, 3.90], + ["ICAC1", 34.008, -118.500, 4, 1, 13.60, 13.20, 8.20, 10.30], + ["ICYA2", 59.923, -141.359, 4, 1, 4.00, null, null, null], + ["IIWC1", 32.714, -117.177, 4, 1, 22.30, 0.60, null, null], + ["ILOH1", 19.730, -155.056, 4, 1, 15.80, 11.80, 4.30, 1.30], + ["IMGP4", 17.969, -67.044, 4, 1, 7.80, 2.00, 2.00, null], + ["IOSN3", 42.967, -70.623, 4, 1, 19.20, 11.90, 18.90, null], + ["JMPN7", 34.213, -77.786, 4, 1, 8.00, 3.20, 9.20, 10.70], + ["KATA1", 30.258, -88.213, 4, 1, 15.65, 4.35, 3.80, 4.00], + ["KDAA2", 57.731, -152.511, 4, 1, 8.50, 7.80, 6.70, 2.90], + ["KGCA2", 55.062, -162.327, 4, 1, 6.90, 6.40, 5.00, 3.20], + ["KLIH1", 20.895, -156.469, 4, 1, 6.90, 6.40, 4.40, 1.50], + ["KNSW3", 42.589, -87.809, 5, 1, 20.00, 16.20, null, null], + ["KTNF1", 29.819, -83.594, 4, 1, 10.00, 9.10, 3.00, null], + ["KWHH1", 20.037, -155.829, 4, 1, 8.20, 6.40, 3.90, 1.60], + ["KWNW3", 44.465, -87.496, 5, 1, 9.30, 6.40, 181.40, null], + ["KYWF1", 24.556, -81.808, 4, 1, 15.00, 3.00, 4.60, 2.40], + ["LDLC3", 41.306, -72.077, 4, 1, 20.00, 20.00, 20.00, null], + ["LJAC1", 32.867, -117.257, 4, 1, 8.20, 7.20, 11.30, 3.40], + ["LKWF1", 26.613, -80.034, 4, 1, 6.00, 4.50, 7.10, 1.60], + ["LMFS1", 34.107, -81.271, 5, 1, 9.00, 3.00, 109.40, null], + ["LONF1", 24.844, -80.864, 4, 1, 6.34, 6.10, 5.03, null], + ["LOPL1", 28.885, -90.025, 3, 1, 57.90, null, 40.50, null], + ["LTBV3", 17.695, -64.754, 4, 1, 6.60, 3.80, 5.10, 1.50], + ["LWSD1", 38.783, -75.119, 4, 1, 9.50, 8.00, 5.20, 1.70], + ["MBET2", 28.422, -96.327, 4, 1, 12.00, 12.00, 11.00, 1.00], + ["MBLA1", 30.437, -88.011, 3, 1, 14.20, 7.34, 6.38, 1.00], + ["MCGA1", 30.649, -88.058, 4, 1, 7.00, 6.60, 3.40, 2.60], + ["MCGM4", 46.546, -87.379, 5, 1, 8.60, 5.10, 187.80, null], + ["MCYI3", 41.729, -86.912, 5, 1, 21.30, 18.30, 193.80, null], + ["MDRM1", 43.969, -68.128, 3, 1, 22.60, 15.20, 16.50, null], + ["MEEM4", 44.251, -86.342, 5, 1, 12.20, 2.00, 179.90, null], + ["MEYC1", 44.251, -86.342, 4, 1, 8.50, 7.90, 5.50, 2.10], + ["MGIP4", 17.970, -67.046, 4, 1, 6.60, 2.80, 2.30, 0.00], + ["MGZP4", 18.218, -67.159, 4, 1, 8.40, 5.00, 3.10, 2.10], + ["MISM1", 43.784, -68.855, 4, 1, 7.10, 3.80, 16.80, null], + ["MIST2", 27.838, -97.050, 4, 1, 5.60, 4.70, 1.20, null], + ["MKGM4", 43.227, -86.339, 5, 1, 24.40, 6.10, 180.90, null], + ["MLRF1", 25.012, -80.376, 3, 1, 15.80, 15.50, 11.30, 1.00], + ["MLWW3", 43.005, -87.884, 5, 1, 7.30, 6.10, null, null], + ["MNMM4", 45.096, -87.590, 5, 1, 10.50, 4.10, 179.60, 5.00], + ["MOKH1", 21.433, -157.790, 4, 1, 10.00, 2.80, 2.80, 1.20], + ["MQTT2", 27.581, -97.217, 4, 1, 7.90, 3.90, 8.70, 1.70], + ["MRHO1", 41.544, -82.731, 5, 1, 9.50, 3.70, 177.50, 4.00], + ["MROS1", 33.656, -78.916, 4, 1, 6.90, 3.70, 5.20, 4.10], + ["MRSL1", 29.441, -92.061, 3, 1, 20.00, 20.00, 12.00, 2.00], + ["MRYA2", 55.099, -131.182, 4, 1, 4.00, null, null, null], + ["MTYC1", 36.605, -121.889, 4, 1, 8.50, 7.90, 5.50, 2.10], + ["NABM4", 46.087, -85.444, 5, 1, 10.00, 3.00, 180.00, null], + ["NFBF1", 25.084, -81.096, 4, 1, 5.50, 3.40, 3.40, 0.60], + ["NKTA2", 60.683, -151.399, 4, 1, 12.40, 3.40, 9.00, 0.60], + ["NMTA2", 64.500, -165.430, 4, 1, 7.40, 4.00, 6.50, 5.00], + ["NPDW3", 45.291, -86.977, 5, 1, 10.00, 10.00, null, null], + ["NSTP6", -14.280, -170.688, 4, 1, 10.70, 9.80, 4.70, 2.20], + ["NTBC1", 34.405, -119.692, 4, 1, 13.20, 12.40, 8.70, null], + ["NTKM3", 41.285, -70.096, 4, 1, 8.50, 3.20, 4.00, 0.70], + ["NWPO3", 44.613, -124.067, 4, 1, 9.40, 6.40, 11.00, null], + ["NWWH1", 21.954, -159.353, 4, 1, 7.10, 1.90, 2.80, 1.70], + ["OCIM2", 38.328, -75.091, 4, 1, 8.50, 2.40, 3.10, 3.70], + ["OLCN6", 43.341, -78.719, 5, 1, 10.00, 8.00, 83.00, null], + ["OLSA2", 52.940, -168.870, 4, 1, 8.40, 7.80, 7.30, null], + ["ORIN7", 35.796, -75.548, 4, 1, 6.70, 5.20, 4.10, 0.80], + ["OSGN6", 43.464, -76.511, 5, 1, 14.90, 10.50, 78.20, null], + ["PCBF1", 30.213, -85.880, 4, 1, 8.60, 3.30, 9.10, null], + ["PCLM4", 47.276, -88.528, 5, 1, 10.00, 2.70, 198.40, null], + ["PCLM4", 33.735, -118.241, 4, 1, 10.70, null, null, null], + ["PFXC1", 33.747, -118.215, 4, 1, 22.90, 22.90, null, null], + ["PGBP7", 13.428, 144.796, 4, 1, 14.30, 10.50, null, null], + ["PILA2", 59.742, -149.470, 4, 1, 6.60, 6.40, 25.00, null], + ["PILM4", 48.223, -88.366, 5, 1, 15.50, 9.80, 230.40, null], + ["PLSF1", 24.693, -82.773, 3, 1, 17.70, 17.40, 15.80, 1.50], + ["PLXA2", 56.247, -134.647, 4, 1, 8.50, 7.90, 5.60, 4.00], + ["PMOA2", 55.990, -160.562, 4, 1, 13.20, 7.30, 5.40, 1.90], + ["PNGW3", 46.792, -91.386, 5, 1, 10.00, 3.00, 185.90, null], + ["PNLM4", 45.968, -85.869, 5, 1, 10.30, 6.50, 180.20, 4.60], + ["PORO3", 42.739, -124.498, 4, 1, 9.10, 8.70, 8.00, 1.30], + ["PRDA2", 70.400, -148.527, 4, 1, null, 23.68, 4.80, 3.30], + ["PSCM4", 43.420, -82.540, 5, 1, 10.00, 3.30, 191.80, null], + ["PSLC1", 35.169, -120.754, 4, 1, 13.60, 12.70, 11.60, 3.50], + ["PSTL1", 28.932, -89.407, 4, 1, 20.30, 2.50, 4.70, 5.70], + ["PTAT2", 27.826, -97.051, 4, 1, 14.90, 9.10, 6.40, 1.00], + ["PTAW1", 48.125, -123.441, 4, 1, 12.20, 4.60, 5.50, 2.90], + ["PTBM6", 30.213, -88.505, 4, 1, 4.60, 2.50, 1.50, null], + ["PTGC1", 34.577, -120.648, 4, 1, 9.40, 9.10, 33.50, null], + ["PTIM4", 46.485, -84.631, 5, 1, 8.90, 6.60, 186.50, null], + ["PTRP4", 18.367, -67.251, 4, 1, 15.00, null, null, null], + ["PTWW1", 48.111, -122.760, 4, 1, 8.90, 2.40, 5.10, 2.80], + ["PVGF1", 26.092, -80.109, 4, 1, 3.50, 3.50, 3.50, 1.65], + ["PWAW3", 43.388, -87.867, 5, 1, 10.00, 2.10, 179.80, null], + ["RDDA2", 67.575, -164.067, 4, 1, 12.70, 9.30, 15.60, 3.10], + ["RIXA2", 58.177, -135.052, 4, 1, 2.70, 2.70, 16.50, null], + ["ROAM4", 47.867, -89.313, 5, 1, 46.90, 46.50, 222.70, null], + ["SACV4", 19.174, -96.093, 4, 1, 9.00, 9.00, 0.00, null], + ["SANF1", 24.456, -81.877, 3, 1, 14.80, 14.60, 13.60, null], + ["SAUF1", 29.857, -81.265, 4, 1, 7.60, 7.60, 8.80, 2.00], + ["SBBN2", 36.050, -114.748, 5, 1, 3.00, 3.00, 330.57, 0.50], + ["SBIO1", 41.629, -82.841, 5, 1, 24.30, 23.40, 178.40, null], + ["SBLM4", 43.810, -83.720, 5, 1, 13.00, 10.30, 188.10, null], + ["SDHN4", 40.467, -74.009, 4, 1, 5.50, 5.10, 2.40, 0.50], + ["SGNW3", 43.749, -87.693, 5, 1, 19.20, 15.50, 189.00, 1.00], + ["SHBL1", 29.868, -89.673, 4, 1, 15.60, 15.00, 8.90, 1.00], + ["SHPF1", 30.058, -84.291, 4, 1, 5.70, 11.90, 5.40, 0.60], + ["SHXA2", 57.055, -135.349, 4, 1, 10.70, 10.70, 15.20, null], + ["SIPF1", 27.862, -80.445, 4, 1, 1.00, 0.30, 1.00, 8.53], + ["SJNP4", 18.459, -66.116, 4, 1, 5.70, 2.80, 3.00, 3.80], + ["SJOM4", 42.098, -86.494, 5, 1, 10.00, 3.30, 184.10, null], + ["SJSN4", 39.305, -75.377, 4, 1, 14.80, 3.00, 7.40, 0.67], + ["SMKF1", 24.628, -81.109, 3, 1, 7.20, 7.00, 6.00, null], + ["SPGF1", 26.704, -78.995, 4, 1, 6.60, 6.10, 2.70, null], + ["SPLL1", 26.704, -78.995, 3, 1, 10.00, 10.00, 3.00, 1.00], + ["SRLM4", 45.773, -84.137, 5, 1, 30.00, null, null, null], + ["STDM4", 47.184, -87.225, 5, 1, 35.20, 35.20, 211.20, null], + ["STXA2", 57.116, -135.391, 4, 1, 9.40, 9.40, 20.10, null], + ["SVNM4", 42.401, -86.288, 5, 1, 16.80, 13.70, null, null], + ["SXHW3", 46.563, -90.437, 5, 1, 10.00, 2.70, 187.00, null], + ["TAWM4", 44.254, -83.449, 5, 1, 10.00, 1.80, 185.30, null], + ["TBIM4", 44.254, -83.449, 5, 1, 10.70, 6.10, null, null], + ["THLO1", 41.826, -83.194, 5, 1, 14.90, 14.00, null, null], + ["TOKW1", 46.707, -123.967, 4, 1, 8.10, 6.70, 4.90, 1.30], + ["TRDF1", 28.416, -80.593, 4, 1, 6.70, 2.50, 5.20, 1.30], + ["TWCO1", 28.416, -80.593, 5, 1, 18.20, 17.50, 189.00, null], + ["TXPT2", 28.416, -80.593, 4, 1, 12.50, 11.60, 11.40, 0.52], + ["VAKF1", 25.731, -80.162, 4, 1, 10.20, 5.00, 4.50, 1.80], + ["VBBA3", 36.132, -114.412, 5, 1, 3.00, 3.00, 330.57, 0.50], + ["VCAF1", 24.711, -81.107, 4, 1, 6.50, 1.70, 4.40, 1.50], + ["VCVA2", 57.125, -170.285, 4, 1, 8.50, 8.00, 5.50, 3.00], + ["VENF1", 27.072, -82.453, 4, 1, 11.60, 7.30, 4.90, 1.00], + ["VQSP4", 18.153, -65.444, 4, 1, 6.40, 3.40, null, null], + ["WAKP8", 19.291, 166.618, 4, 1, 6.20, 4.90, 4.80, 1.00], + ["WELM1", 43.320, -70.563, 4, 1, 7.20, 3.90, 4.80, 1.80], + ["WFPM4", 46.760, -84.970, 5, 1, 16.76, 2.13, 190.19, null], + ["WHRI2", 42.361, -87.813, 5, 1, 9.00, 7.90, 187.80, null], + ["WPTW1", 46.904, -124.105, 4, 1, 26.80, 7.30, 8.00, 1.90], + ["WYCM6", 30.326, -89.326, 4, 1, 9.90, 9.30, 8.50, 0.20], + ["YABP4", 18.055, -65.833, 4, 1, 6.40, 3.40, null, null], + ["YGNN6", 43.262, -79.064, 5, 1, 10.00, 6.00, 75.30, null] + ] + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/GeometryUtilTest.java b/core/src/test/java/com/bc/fiduceo/GeometryUtilTest.java index f3f2de51c..dbb5fec91 100644 --- a/core/src/test/java/com/bc/fiduceo/GeometryUtilTest.java +++ b/core/src/test/java/com/bc/fiduceo/GeometryUtilTest.java @@ -24,6 +24,8 @@ import com.bc.fiduceo.geometry.*; import org.junit.Test; +import java.util.Arrays; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -68,6 +70,68 @@ public void testToKml_simplePolygon() { "", GeometryUtil.toKml(polygon)); } + @Test + public void testToKml_multiPolygon() { + final GeometryFactory geometryFactory = new GeometryFactory(GeometryFactory.Type.S2); + final Polygon polygon1 = (Polygon) geometryFactory.parse("POLYGON((0 0, 0 1, 1 1, 1 0, 0 0))"); + final Polygon polygon2 = (Polygon) geometryFactory.parse("POLYGON((2 2, 2 3, 3 3, 3 2, 2 2))"); + + final MultiPolygon multiPolygon = geometryFactory.createMultiPolygon(Arrays.asList(polygon1, polygon2)); + + final String expectedKml = + "\n" + + "\n" + + "\n" + + " \n" + + " \n" + + " part_1\n" + + " #polygonStyle\n" + + " \n" + + " clampToGround\n" + + " \n" + + " \n" + + " \n" + + " 0.0,0.0,0\n" + + " 0.0,1.0,0\n" + + " 0.9999999999999998,1.0,0\n" + + " 1.0,0.0,0\n" + + " 0.0,0.0,0\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " part_2\n" + + " #polygonStyle\n" + + " \n" + + " clampToGround\n" + + " \n" + + " \n" + + " \n" + + " 2.0,1.9999999999999996,0\n" + + " 1.9999999999999996,3.0000000000000004,0\n" + + " 3.0000000000000004,3.000000000000001,0\n" + + " 3.0000000000000004,2.0,0\n" + + " 2.0,1.9999999999999996,0\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n" + + ""; + assertEquals(expectedKml, GeometryUtil.toKml(multiPolygon)); + } + @Test public void testCreatePolygonFromMinMax() { final double[] geoMinMax = {-170.0, 170.0, -80.0, 80.0}; diff --git a/core/src/test/java/com/bc/fiduceo/NCTestUtils.java b/core/src/test/java/com/bc/fiduceo/NCTestUtils.java index 49d732063..d48df439f 100644 --- a/core/src/test/java/com/bc/fiduceo/NCTestUtils.java +++ b/core/src/test/java/com/bc/fiduceo/NCTestUtils.java @@ -208,25 +208,25 @@ public static Variable getVariable(String variableName, NetcdfFile netcdfFile, b } public static void assert2DValueInt(int x, int y, int expected, Variable variable) throws IOException, InvalidRangeException { - assertNotNull("NetCDF Variable '" + variable.getShortName() + "' expected", variable); + assertNotNull("NetCDF Variable expected", variable); final Array data = variable.read(new int[]{y, x}, new int[]{1, 1}); assertEquals(expected, data.getInt(0)); } public static void assert2DValueFloat(int x, int y, float expected, Variable variable) throws IOException, InvalidRangeException { - assertNotNull("NetCDF Variable '" + variable.getShortName() + "' expected", variable); + assertNotNull("NetCDF Variable expected", variable); final Array data = variable.read(new int[]{y, x}, new int[]{1, 1}); assertEquals(expected, data.getFloat(0), 1e-8); } public static void assert3DValueDouble(int x, int y, int z, double expected, Variable variable) throws IOException, InvalidRangeException { - assertNotNull("NetCDF Variable '" + variable.getShortName() + "' expected", variable); + assertNotNull("NetCDF Variable expected", variable); final Array data = variable.read(new int[]{z, y, x}, new int[]{1, 1, 1}); assertEquals(expected, data.getDouble(0), 1e-8); } public static void assert1DValueLong(int x, long expected, Variable variable) throws IOException { - assertNotNull("NetCDF Variable '" + variable.getShortName() + "' expected", variable); + assertNotNull("NetCDF Variable expected", variable); final Array array = variable.read(); final Index index = array.getIndex(); index.set(x); @@ -234,7 +234,7 @@ public static void assert1DValueLong(int x, long expected, Variable variable) th } public static void assert3DValueLong(int x, int y, int z, long expected, Variable variable) throws IOException, InvalidRangeException { - assertNotNull("NetCDF Variable '" + variable.getShortName() + "' expected", variable); + assertNotNull("NetCDF Variable expected", variable); final Array data = variable.read(new int[]{z, y, x}, new int[]{1, 1, 1}); assertEquals(expected, data.getLong(0)); } diff --git a/core/src/test/java/com/bc/fiduceo/TestData.java b/core/src/test/java/com/bc/fiduceo/TestData.java index 1b483ba98..e6f05982b 100644 --- a/core/src/test/java/com/bc/fiduceo/TestData.java +++ b/core/src/test/java/com/bc/fiduceo/TestData.java @@ -63,6 +63,12 @@ public class TestData { public static final String[] MHS_N18_AXIS_GEOMETRIES = new String[]{"LINESTRING(5.287599866423989 66.96359830835718,-3.276899917218543 72.58319816639414,-18.611199529841546 77.58849803994964,-48.169098783146175 80.90899795606674,-88.22379777127935 80.5615979648428,-114.92089709685388 76.83389805901243,-128.7164967483477 71.68349818912247,-136.5986965492266 66.00759833250777,-141.72929641961673 60.101298481713464,-145.4234963262934 54.07559863393544,-148.29069625386182 47.97959878793336,-150.645796194367 41.83819894307817,-152.6668961433097 35.66549909901369,-154.46249609794904 29.4706992555075,-156.10359605649137 23.259999412402976,-157.6389960177039 17.03859956956876,-159.10499598066966 10.810799726896223,-160.53019594466605 4.58029988429189,-161.9388959090793 -1.6491999583377042,-163.35399587333086 -7.87399980108603,-164.79889583682962 -14.09079964403645,-166.2997957989137 -20.295999487279907,-167.88849575877975 -26.485599330917466,-169.60679571537185 -32.65549917505268,-171.51309566721466 -38.80049901981693,-173.69389561212301 -44.91339886539208,-176.2858955466436 -50.98349871204846,-179.51999546494338 -56.992798560240765,176.18179554927337 -62.90759841082035,169.96809570624464 -68.65739826556819,161.8110959123078 -73.29379814844287)", "LINESTRING(161.8110959123078 -73.29379814844287,145.09969633447326 -78.07099802776065,113.85819712369995 -80.953797954935,75.33869809678436 -80.11889797602635,50.85769871522645 -76.22539807438443,37.9635990409588 -71.07199820457026,30.421899231478175 -65.43499834697286,25.43409935748059 -59.57729849495081,21.805299449151782 -53.600498645937485,18.969899520780018 -47.55039879877586,16.630599579875707 -41.45039895287482,14.617399630733415 -35.31389910789585,12.82529967600567 -29.14919926362927,11.18569971742545 -22.962399419920988,9.650399756210389 -16.757999576657312,8.18399979325477 -10.540099733734682,6.7581998292735115 -4.31259989105456,5.3484998648855235 1.9210999514689322,3.9322999006617465 8.157399793926746,2.485899937200883 14.392499636414868,0.9827999751723838 20.622699479026778,-0.6094999846027348 26.8436993218711,-2.333799941043253 33.05089916506404,-4.250399892625865 39.23849900875211,-6.449399837074452 45.39869885313237,-9.074399770761374 51.51979869850038,-12.371199687477205 57.58249854534369,-16.796299575689773 63.55009839458945,-23.28959941165522 69.34509824819543,-34.02059914056736 74.77589811100188,-49.64379874589213 78.61029801413679)"}; + public static final String[] MHS_MC_GEOMETRIES = new String[] {"POLYGON((84.0214 56.086600000000004,78.9127 57.7984,74.9634 58.8833,71.806 59.61880000000001,68.9829 60.1865,66.2131 60.668,63.24199999999999 61.107600000000005,59.7302 61.532,55.042 61.94789999999999,47.4992 62.275800000000004,47.4154 61.19950000000001,47.1425 58.1229,46.828799999999994 55.0437,46.4826 51.96189999999999,46.1093 48.8777,45.712700000000005 45.79090000000001,45.29540000000001 42.7018,44.8591 39.6103,44.4049 36.51669999999999,43.9336 33.421,43.44530000000001 30.32360000000001,42.9402 27.2246,42.41770000000001 24.1244,41.8776 21.0233,41.31880000000001 17.9217,40.7406 14.82,40.1417 11.7187,39.520500000000006 8.6182,38.8756 5.5193,38.2047 2.4222999999999995,37.5057 -0.6719,36.7758 -3.7626,36.012 -6.849099999999999,35.2107 -9.9304,34.367700000000006 -13.0056,33.47820000000001 -16.0736,32.53649999999999 -19.1333,31.5359 -22.1831,30.4687 -25.2216,29.3253 -28.247,28.094799999999996 -31.257199999999997,26.7638 -34.2495,25.3161 -37.2211,23.732099999999996 -40.1682,21.987700000000004 -43.086400000000005,20.053300000000004 -45.97,17.8924 -48.8121,15.4591 -51.6036,12.696399999999999 -54.333,9.533300000000002 -56.9851,5.8817 -59.54,1.6346000000000003 -61.971700000000006,-3.3334 -64.2455,-9.158 -66.3163,-15.964799999999997 -68.1266,-23.8255 -69.6074,-32.6925 -70.6829,-42.3319 -71.2834,-52.315 -71.3628,-62.1096 -70.9148,-71.2418 -69.9746,-79.4159 -68.6069,-86.5346 -66.8871,-92.6413 -64.8874,-97.85120000000002 -62.66859999999999,-102.30010000000001 -60.2796,-106.1178 -57.7579,-109.41700000000002 -55.1319,-114.51450000000001 -56.8631,-118.4301 -57.96479999999999,-121.54979999999999 -58.7174,-124.3346 -59.302800000000005,-127.0657 -59.8026,-129.9965 -60.2609,-133.4653 -60.70420000000001,-138.1052 -61.139100000000006,-145.5978 -61.4848,-145.53340000000003 -62.556400000000004,-145.3999 -65.6158,-145.3651 -68.6719,-145.476 -71.7241,-145.8175 -74.7719,-146.5595 -77.8136,-148.0966 -80.8453,-151.592 -83.8539,-162.4172 -86.7779,127.1743 -88.728,63.6676 -86.612,53.7031 -83.6781,50.391 -80.6675,48.9202 -77.6355,48.2092 -74.59360000000001,47.88479999999999 -71.546,47.78430000000001 -68.494,47.8259 -65.4385,47.964 -62.3794,48.1715 -59.3172,48.4311 -56.2518,48.731500000000004 -53.18310000000001,49.06519999999999 -50.1112,49.4268 -47.0361,49.81270000000001 -43.957800000000006,50.2203 -40.8763,50.648 -37.7918,51.0945 -34.7044,51.5591 -31.6143,52.04180000000001 -28.5216,52.5423 -25.4268,53.0611 -22.329900000000002,53.5987 -19.2315,54.15590000000001 -16.1318,54.7337 -13.0314,55.3334 -9.9308,55.9566 -6.8304,56.605 -3.7307999999999995,57.28079999999999 -0.6328,57.9865 2.4630000000000005,58.72480000000001 5.5558000000000005,59.49920000000001 8.6449,60.3136 11.729100000000003,61.17240000000001 14.8074,62.0809 17.8788,63.045500000000004 20.9419,64.0734 23.9952,65.1733 27.037,66.3556 30.065199999999997,67.6328 33.07770000000001,69.0198 36.07139999999999,70.535 39.043,72.2007 41.9885,74.0444 44.9026,76.1001 47.7789,78.4105 50.6092,81.02880000000002 53.383,84.0214 56.086600000000004))", + "POLYGON((-145.5978 -61.4848,-138.6796 -61.18079999999999,-133.8614 -60.74829999999999,-130.3121 -60.3056,-127.34530000000001 -59.8497,-124.606 -59.355700000000006,-121.83799999999998 -58.7816,-118.7677 -58.0514,-114.96390000000002 -56.999,-109.41700000000002 -55.1319,-110.4671 -54.1924,-113.21150000000002 -51.4589,-115.6297 -48.6638,-117.7781 -45.8186,-119.702 -42.9322,-121.43749999999999 -40.0116,-123.01410000000001 -37.0621,-124.45550000000001 -34.0884,-125.78099999999999 -31.094,-127.00680000000001 -28.082,-128.146 -25.0547,-129.2097 -22.014400000000002,-130.2071 -18.9628,-131.146 -15.901400000000004,-132.033 -12.831700000000001,-132.8737 -9.7549,-133.673 -6.6720000000000015,-134.435 -3.5839,-135.1633 -0.4915999999999999,-135.8608 2.6041,-136.5303 5.702499999999999,-137.174 8.8031,-137.7939 11.9049,-138.3918 15.0077,-138.969 18.110700000000005,-139.5267 21.2137,-140.066 24.316100000000002,-140.58750000000003 27.4175,-141.0918 30.517699999999998,-141.5792 33.616299999999995,-142.0496 36.713,-142.5028 39.8077,-142.9381 42.90010000000001,-143.3543 45.99020000000001,-143.7497 49.0778,-144.1216 52.1629,-144.4661 55.2453,-144.7776 58.3253,-145.0478 61.40250000000001,-145.2641 64.4772,-145.4065 67.5491,-145.4425 70.6183,-145.3144 73.6843,-144.911 76.7464,-143.9878 79.8024,-141.8847 82.8465,-136.033 85.8546,-102.84770000000002 88.5814,1.6634999999999998 87.6229,16.191 84.6991,19.9666 81.6714,21.479499999999998 78.6217,22.1571 75.5633,22.435200000000002 72.5,22.4906 69.4332,22.4084 66.3635,22.234399999999997 63.29110000000001,21.9955 60.21620000000001,28.5122 59.9161,33.07630000000001 59.5112,36.4555 59.102900000000005,39.293000000000006 58.6847,41.9232 58.2312,44.590300000000006 57.7019,47.5576 57.0242,51.2439 56.0401,56.6331 54.28379999999999,57.6803 55.2302,60.9742 57.876,64.7932 60.4181,69.25380000000001 62.8276,74.4911 65.0665,80.6475 67.0847,87.8452 68.8195,96.13240000000002 70.1963,105.40760000000002 71.1369,115.35980000000002 71.5743,125.4889 71.473,135.239 70.8415,144.1702 69.7292,152.0532 68.2094,158.8515 66.3605,164.6489 64.2529,169.5799 61.944900000000004,173.7863 59.48149999999999,177.397 56.897200000000005,-179.4793 54.218,-176.7537 51.46360000000001,-174.3548 48.6487,-172.2256 45.7848,-170.3206 42.88080000000001,-168.6034 39.94350000000001,-167.0444 36.9785,-165.62 33.99020000000001,-164.3108 30.9822,-163.1007 27.957500000000003,-161.9766 24.9185,-160.9275 21.867399999999996,-159.9443 18.8059,-159.0192 15.735699999999998,-158.1457 12.658100000000001,-157.3181 9.574300000000001,-156.5319 6.485299999999998,-155.7828 3.3923,-155.0674 0.296,-154.3827 -2.8029,-153.7261 -5.9033999999999995,-153.0954 -9.004899999999997,-152.4887 -12.106900000000001,-151.9043 -15.208799999999998,-151.3409 -18.3101,-150.7974 -21.4103,-150.27290000000002 -24.508899999999997,-149.7669 -27.6057,-149.2788 -30.70030000000001,-148.8087 -33.7925,-148.3567 -36.8819,-147.9233 -39.9684,-147.50950000000003 -43.0519,-147.1169 -46.13219999999999,-146.7478 -49.2093,-146.4054 -52.283100000000005,-146.0945 -55.35360000000001,-145.822 -58.4208,-145.5978 -61.4848))"}; + + public static final String[] MHS_MC_AXIS_GEOMETRIES = new String[] {"LINESTRING(67.3303 60.482400000000005,65.3291 57.465500000000006,63.5992 54.4258,62.077200000000005 51.3684,60.7174 48.2967,59.486200000000004 45.21329999999999,58.358200000000004 42.1202,57.3139 39.0189,56.3382 35.9107,55.4189 32.796400000000006,54.5463 29.677000000000007,53.7124 26.553200000000004,52.91040000000001 23.425700000000003,52.1348 20.294799999999995,51.3808 17.161400000000004,50.6441 14.0259,49.9209 10.8886,49.2079 7.750199999999999,48.502 4.610999999999999,47.8 1.4715999999999998,47.0992 -1.6676000000000002,46.3968 -4.8062,45.689699999999995 -7.943699999999998,44.9751 -11.079700000000003,44.24979999999999 -14.2137,43.5103 -17.345199999999995,42.7529 -20.473800000000004,41.9732 -23.599,41.16640000000001 -26.7201,40.3268 -29.8368,39.4476 -32.94830000000001,38.5206 -36.0538,37.5361 -39.15259999999999,36.4816 -42.2438,35.3418 -45.3259,34.09689999999999 -48.3976,32.7213 -51.4568,31.1808 -54.5009,29.429199999999998 -57.5263,27.4026 -60.527800000000006,25.01 -63.4983,22.1183 -66.4266,18.527099999999997 -69.2959,13.926000000000002 -72.0786,7.822900000000001 -74.7282,-0.5557000000000001 -77.163,-12.3195 -79.2371,-28.517799999999998 -80.7064,-48.51310000000001 -81.2611,-68.5717 -80.7353,-84.8968 -79.2868,-96.7715 -77.2255,-105.2273 -74.798,-111.3811 -72.1528,-116.01600000000002 -69.3727,-119.63020000000002 -66.505,-122.53819999999999 -63.5775,-124.94259999999998 -60.60750000000001,-125.69149999999999 -59.5601)", + "LINESTRING(-125.69149999999999 -59.5601,-127.6211 -56.5496,-129.2986 -53.5171,-130.7816 -50.46710000000001,-132.1119 -47.4027,-133.3204 -44.32670000000001,-134.4309 -41.24060000000001,-135.4613 -38.146,-136.4262 -35.0439,-137.3368 -31.935299999999994,-138.2025 -28.8211,-139.0309 -25.701800000000002,-139.8285 -22.578200000000002,-140.6006 -19.4507,-141.3519 -16.319999999999997,-142.0865 -13.186399999999999,-142.8082 -10.050500000000001,-143.5202 -6.9128,-144.2256 -3.7737000000000003,-144.9274 -0.6337,-145.6284 2.5069,-146.33160000000004 5.6475,-147.0397 8.7876,-147.7559 11.926900000000003,-148.4834 15.0648,-149.2256 18.2009,-149.9865 21.3346,-150.7705 24.4655,-151.5827 27.593,-152.429 30.716400000000007,-153.3165 33.8351,-154.2537 36.94840000000001,-155.2511 40.05539999999999,-156.3218 43.15500000000001,-157.4823 46.245799999999996,-158.7537 49.3264,-160.16400000000002 52.3943,-161.7503 55.4469,-163.5636 58.4801,-165.6749 61.4885,-168.1867 64.4637,-171.2507 67.3935,-175.0984 70.2582,179.9056 73.0259,173.1776 75.6409,163.8013 78.0039,150.5331 79.93660000000001,132.5518 81.1529,111.58700000000002 81.3456,92.192 80.4528,77.3218 78.7293,66.7553 76.489,59.24009999999999 73.9465,53.7291 71.2238,49.53659999999999 68.3882,46.23469999999999 65.4786,43.55340000000001 62.51780000000001,41.318 59.5204,40.618 58.4644)"}; + public static final String[] HIRS_TN_GEOMETRIES = new String[]{"POLYGON((-50.6875 55.8515625,-47.484375 56.96874999999999,-44.2265625 57.96875,-41.6171875 58.6796875,-39.3828125 59.234375,-37.375 59.6875,-35.4765625 60.07812500000001,-33.625 60.42968750000001,-31.742187500000007 60.75,-29.7578125 61.05468750000001,-27.5703125 61.34375000000001,-25.062500000000004 61.62499999999999,-22.015625 61.8984375,-18.046875000000007 62.13281250000001,-12.320312500000004 62.265625,-12.453125 65.5859375,-12.460937500000002 69.2734375,-12.203124999999998 72.9609375,-11.500000000000004 76.6328125,-9.812500000000002 80.296875,-5.4765625 83.9296875,12.562500000000002 87.3828125,117.29687500000001 87.9921875,144.90625 84.671875,150.46875 81.05468750000001,152.3984375 77.40625,153.28125 73.734375,153.5859375 70.0625,153.65625 66.375,153.515625 62.69531249999999,153.2890625 59.00000000000001,152.9609375 55.3203125,152.59375 51.61718749999999,152.1640625 47.9296875,151.7109375 44.2265625,151.203125 40.53906249999999,150.6875 36.828125,150.1171875 33.140625,149.546875 29.4296875,148.9296875 25.734375,148.296875 22.03125,147.625 18.3359375,146.9375 14.632812500000002,146.2109375 10.945312500000004,145.453125 7.250000000000001,144.65625 3.5781249999999996,143.8203125 -0.1015625,142.9375 -3.7656250000000018,142.0078125 -7.4375,141.0234375 -11.078125,139.9765625 -14.726562500000002,138.8515625 -18.34375,137.6484375 -21.9609375,136.3515625 -25.5390625,134.9375 -29.1171875,133.4140625 -32.63281250000001,131.7109375 -36.1484375,129.8359375 -39.60937499999999,127.7265625 -43.04687500000001,125.35937500000001 -46.42187500000001,122.64062500000001 -49.734375,119.53906249999999 -52.9609375,115.8828125 -56.085937500000014,111.62500000000001 -59.07031250000001,115.05468749999999 -60.3203125,118.609375 -61.42187500000001,121.5 -62.1953125,124.00781250000001 -62.78906249999999,126.28125000000001 -63.2578125,128.4375 -63.66406250000001,130.5625 -64.015625,132.71875 -64.328125,135.0078125 -64.625,137.5234375 -64.90625,140.421875 -65.1796875,143.9375 -65.42187500000001,148.515625 -65.6328125,155.1328125 -65.6875,155.21875 -62.3671875,155.45312500000003 -58.6953125,155.75 -55.00781249999999,156.125 -51.328125000000014,156.53125000000003 -47.625,156.9921875 -43.9453125,157.46875 -40.2421875,157.9921875 -36.5546875,158.5390625 -32.8515625,159.125 -29.187499999999996,159.7265625 -25.476562499999993,160.3515625 -21.789062500000007,161.015625 -18.078125,161.6953125 -14.3984375,162.421875 -10.695312500000002,163.171875 -7.015624999999998,163.96875 -3.3203125,164.7890625 0.3437500000000001,165.671875 4.0234375,166.59375 7.679687500000003,167.5859375 11.335937500000002,168.625 14.96875,169.75 18.601562499999993,170.9453125 22.2109375,172.2578125 25.804687500000004,173.65625 29.375,175.2109375 32.92187500000001,176.8984375 36.4296875,178.8046875 39.9140625,-179.0859375 43.34374999999999,-176.6796875 46.7265625,-173.9609375 50.03125,-170.796875 53.26562500000001,-167.1328125 56.390625,-162.7734375 59.37499999999999,-157.6171875 62.1796875,-151.40625 64.7421875,-144.0078125 66.9921875,-135.2109375 68.81250000000001,-125.140625 70.1171875,-114.0234375 70.7578125,-102.5859375 70.70312500000001,-91.6015625 69.93749999999999,-81.6953125 68.546875,-73.140625 66.640625,-65.9375 64.3359375,-59.9296875 61.71875,-54.90625 58.875,-50.6875 55.8515625))", "POLYGON((155.1328125 -65.6875,149.921875 -65.66406250000001,144.9609375 -65.48437500000001,141.2265625 -65.2421875,138.203125 -64.9765625,135.609375 -64.6953125,133.28125 -64.40625,131.09375 -64.09375,128.96875 -63.75000000000001,126.82812500000001 -63.3671875,124.59375 -62.9140625,122.15624999999999 -62.359375,119.38281249999999 -61.640625,116.03124999999999 -60.640625,111.62500000000001 -59.07031250000001,107.0625 -61.61718749999999,101.109375 -64.21875,93.9296875 -66.5234375,85.4765625 -68.4296875,75.625 -69.8203125,64.765625 -70.59375,53.374999999999986 -70.671875,42.3359375 -70.0546875,32.21875 -68.78125,23.4375 -66.9921875,15.984375000000002 -64.7578125,9.7578125 -62.21875000000001,4.546874999999998 -59.421875,0.1718749999999999 -56.453125,-3.5234375 -53.3359375,-6.703125000000002 -50.1171875,-9.4609375 -46.796875,-11.875 -43.4296875,-14.007812500000004 -39.9921875,-15.9140625 -36.51562499999999,-17.625 -32.99999999999999,-19.1875 -29.453125000000004,-20.609375 -25.87500000000001,-21.921875 -22.28125000000001,-23.1328125 -18.656250000000004,-24.265625 -15.023437499999998,-25.320312500000004 -11.375,-26.3125 -7.7109375000000036,-27.2421875 -4.03125,-28.132812499999996 -0.3515625,-28.96875 3.335937500000001,-29.765625000000004 7.031249999999998,-30.5234375 10.734375,-31.257812499999996 14.4453125,-31.953125000000007 18.148437500000004,-32.617187500000014 21.859375,-33.24999999999999 25.5703125,-33.86718750000001 29.2890625,-34.4453125 32.999999999999986,-35.0078125 36.7109375,-35.53125000000001 40.42187500000001,-36.03125 44.1328125,-36.4921875 47.835937499999986,-36.921875 51.5390625,-37.296875 55.2421875,-37.617187500000014 58.9375,-37.85937499999999 62.6328125,-37.984375 66.328125,-37.9921875 67.4296875,-43.484375 67.3671875,-48.68749999999999 67.1328125,-52.578125 66.828125,-55.7109375 66.5078125,-58.38281249999999 66.171875,-60.765625 65.8203125,-62.97656249999999 65.453125,-65.1171875 65.0546875,-67.2578125 64.609375,-69.4921875 64.1015625,-71.9140625 63.48437500000001,-74.671875 62.7109375,-77.984375 61.65625,-82.3203125 60.03125,-81.359375 59.45312500000001,-76.984375 56.46875,-73.2890625 53.3359375,-70.1171875 50.1015625,-67.37500000000001 46.78125,-64.9765625 43.39062499999999,-62.85156250000001 39.95312500000001,-60.953125 36.46875,-59.25000000000001 32.9453125,-57.6953125 29.3828125,-56.28125000000001 25.804687500000004,-54.9765625 22.195312500000007,-53.7734375 18.578124999999993,-52.6484375 14.9375,-51.6015625 11.281249999999996,-50.6171875 7.617187499999997,-49.6875 3.9375000000000018,-48.8046875 0.24999999999999992,-47.96875 -3.4296874999999996,-47.1796875 -7.1328125,-46.42187500000001 -10.828125000000002,-45.703125000000014 -14.539062500000007,-45.0078125 -18.2421875,-44.3515625 -21.953124999999996,-43.718750000000014 -25.664062499999996,-43.1171875 -29.375,-42.5390625 -33.08593750000001,-41.99999999999999 -36.78906250000001,-41.4765625 -40.5,-41.0 -44.1953125,-40.5390625 -47.89843750000001,-40.1328125 -51.59375000000001,-39.76562500000001 -55.28906250000001,-39.4765625 -58.9765625,-39.2578125 -62.67187500000001,-39.171875 -66.3515625,-39.25000000000001 -70.03125,-39.65625 -73.703125,-40.578125 -77.3671875,-42.7734375 -81.0078125,-48.6015625 -84.6171875,-76.5703125 -87.890625,-177.7734375 -87.3671875,162.984375 -83.9609375,158.15625 -80.34375,156.3359375 -76.703125,155.5 -73.03125,155.1953125 -69.375,155.1328125 -65.6875))"}; diff --git a/core/src/test/java/com/bc/fiduceo/TestUtil.java b/core/src/test/java/com/bc/fiduceo/TestUtil.java index d9fc711d4..4324beba3 100644 --- a/core/src/test/java/com/bc/fiduceo/TestUtil.java +++ b/core/src/test/java/com/bc/fiduceo/TestUtil.java @@ -29,10 +29,10 @@ import org.hamcrest.Factory; import org.hamcrest.Matcher; import org.hamcrest.TypeSafeMatcher; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.JDOMException; -import org.jdom.input.SAXBuilder; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.JDOMException; +import org.jdom2.input.SAXBuilder; import java.io.*; import java.nio.file.Files; @@ -161,6 +161,9 @@ public static void writeSystemConfig(File configDir, File archiveRoot) throws IO " \n" + " insitu/ndbc/SENSOR/VERSION/YEAR\n" + " " + + " \n" + + " insitu/SENSOR/VERSION/YEAR/MONTH\n" + + " " + " " + " " + " " + TestUtil.getTestDir().getAbsolutePath() + diff --git a/core/src/test/java/com/bc/fiduceo/core/IntRangeTest.java b/core/src/test/java/com/bc/fiduceo/core/IntRangeTest.java new file mode 100644 index 000000000..93e32b59a --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/core/IntRangeTest.java @@ -0,0 +1,54 @@ +package com.bc.fiduceo.core; + +import org.junit.Test; + +import static org.junit.Assert.*; + +public class IntRangeTest { + + @Test + public void testConstruction() { + final IntRange intRange = new IntRange(2, 7); + + assertEquals(2, intRange.getMin()); + assertEquals(7, intRange.getMax()); + } + + @Test + public void testDefaultConstruction() { + final IntRange intRange = new IntRange(); + + assertEquals(Integer.MAX_VALUE, intRange.getMin()); + assertEquals(Integer.MIN_VALUE, intRange.getMax()); + } + + @Test + public void testSetGetMinMax() { + final IntRange intRange = new IntRange(4, 8); + + assertEquals(4, intRange.getMin()); + intRange.setMin(5); + assertEquals(5, intRange.getMin()); + + assertEquals(8, intRange.getMax()); + intRange.setMax(6); + assertEquals(6, intRange.getMax()); + } + + @Test + public void testGetLength() { + final IntRange intRange = new IntRange(3, 8); + + assertEquals(6, intRange.getLength()); + } + + @Test + public void testContains() { + final IntRange intRange = new IntRange(4, 9); + + assertFalse(intRange.contains(3)); + assertTrue(intRange.contains(4)); + assertTrue(intRange.contains(9)); + assertFalse(intRange.contains(10)); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/core/UseCaseConfigBuilder.java b/core/src/test/java/com/bc/fiduceo/core/UseCaseConfigBuilder.java index cbbc439c2..2b71c093a 100644 --- a/core/src/test/java/com/bc/fiduceo/core/UseCaseConfigBuilder.java +++ b/core/src/test/java/com/bc/fiduceo/core/UseCaseConfigBuilder.java @@ -16,9 +16,9 @@ */ package com.bc.fiduceo.core; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.output.XMLOutputter; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.output.XMLOutputter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/core/src/test/java/com/bc/fiduceo/core/UseCaseConfigTest.java b/core/src/test/java/com/bc/fiduceo/core/UseCaseConfigTest.java index 33e58ad5b..929997418 100644 --- a/core/src/test/java/com/bc/fiduceo/core/UseCaseConfigTest.java +++ b/core/src/test/java/com/bc/fiduceo/core/UseCaseConfigTest.java @@ -20,7 +20,7 @@ package com.bc.fiduceo.core; -import org.jdom.JDOMException; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolIntegrationTest.java b/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolIntegrationTest.java index 498d6ab46..72f7b1cf8 100644 --- a/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolIntegrationTest.java +++ b/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolIntegrationTest.java @@ -61,7 +61,7 @@ public void testInvalidCommandLine() throws ParseException { psE.flush(); assertEquals("", out.toString()); - assertEquals("db-maintenance-tool version 1.5.8" + ls + + assertEquals("db-maintenance-tool version 1.6.2" + ls + ls + "usage: db_maintenance " + ls + "Valid options are:" + ls + diff --git a/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolTest.java b/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolTest.java index ac82f3f06..aca9e9f20 100644 --- a/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolTest.java +++ b/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolTest.java @@ -59,7 +59,7 @@ public void testPrintUsageTo() { tool.printUsageTo(stream); - assertEquals("db-maintenance-tool version 1.5.8" + ls + + assertEquals("db-maintenance-tool version 1.6.2" + ls + ls + "usage: db_maintenance " + ls + "Valid options are:" + ls + diff --git a/core/src/test/java/com/bc/fiduceo/geometry/BcGeometryCollectionTest.java b/core/src/test/java/com/bc/fiduceo/geometry/BcGeometryCollectionTest.java index c2a7b6ce2..3bd67158e 100644 --- a/core/src/test/java/com/bc/fiduceo/geometry/BcGeometryCollectionTest.java +++ b/core/src/test/java/com/bc/fiduceo/geometry/BcGeometryCollectionTest.java @@ -2,10 +2,9 @@ import com.bc.fiduceo.geometry.s2.BcS2GeometryFactory; import org.junit.Test; +import org.mockito.Mockito; import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class BcGeometryCollectionTest { @Test @@ -38,10 +37,10 @@ public void testGetInner() { public void testIsValid_valid() { final GeometryCollection geometryCollection = new BcGeometryCollection(); final Geometry[] geometries = new Geometry[2]; - geometries[0] = mock(Geometry.class); - when(geometries[0].isValid()).thenReturn(true); - geometries[1] = mock(Geometry.class); - when(geometries[1].isValid()).thenReturn(true); + geometries[0] = Mockito.mock(Geometry.class); + Mockito.when(geometries[0].isValid()).thenReturn(true); + geometries[1] = Mockito.mock(Geometry.class); + Mockito.when(geometries[1].isValid()).thenReturn(true); geometryCollection.setGeometries(geometries); assertTrue(geometryCollection.isValid()); @@ -51,10 +50,10 @@ public void testIsValid_valid() { public void testIsValid_invalid() { final GeometryCollection geometryCollection = new BcGeometryCollection(); final Geometry[] geometries = new Geometry[2]; - geometries[0] = mock(Geometry.class); - when(geometries[0].isValid()).thenReturn(false); - geometries[1] = mock(Geometry.class); - when(geometries[1].isValid()).thenReturn(true); + geometries[0] = Mockito.mock(Geometry.class); + Mockito.when(geometries[0].isValid()).thenReturn(false); + geometries[1] = Mockito.mock(Geometry.class); + Mockito.when(geometries[1].isValid()).thenReturn(true); geometryCollection.setGeometries(geometries); assertFalse(geometryCollection.isValid()); diff --git a/core/src/test/java/com/bc/fiduceo/geometry/GeometryTestUtils.java b/core/src/test/java/com/bc/fiduceo/geometry/GeometryTestUtils.java new file mode 100644 index 000000000..d97106792 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/geometry/GeometryTestUtils.java @@ -0,0 +1,16 @@ +package com.bc.fiduceo.geometry; + +import static org.junit.Assert.assertEquals; + +public class GeometryTestUtils { + + public static void assertSameGeometry(Geometry expected, Geometry geoBounds) { + Point[] expectedCoords = expected.getCoordinates(); + Point[] actualCoords = geoBounds.getCoordinates(); + assertEquals(expectedCoords.length, actualCoords.length); + for(int i = 0; i < expectedCoords.length; i++) { + assertEquals(expectedCoords[i].getLon(), actualCoords[i].getLon(), 1e-8); + assertEquals(expectedCoords[i].getLat(), actualCoords[i].getLat(), 1e-8); + } + } +} diff --git a/core/src/test/java/com/bc/fiduceo/geometry/s2/BcS2GeometryFactoryTest.java b/core/src/test/java/com/bc/fiduceo/geometry/s2/BcS2GeometryFactoryTest.java index d4def467b..581d28a8d 100644 --- a/core/src/test/java/com/bc/fiduceo/geometry/s2/BcS2GeometryFactoryTest.java +++ b/core/src/test/java/com/bc/fiduceo/geometry/s2/BcS2GeometryFactoryTest.java @@ -120,7 +120,7 @@ public void testFormat_point() { final Point point = factory.createPoint(34.8, -72.44); final String wkt = factory.format(point); - assertEquals("POINT(34.8,-72.44)", wkt); + assertEquals("POINT(34.8 -72.44)", wkt); } @Test @@ -267,7 +267,7 @@ public void testToStorageFormat_point() { final Geometry point = factory.parse("POINT(-22.5 67.23)"); final byte[] storageFormat = factory.toStorageFormat(point); - assertEquals("POINT(-22.500000000000004,67.23)", new String(storageFormat)); + assertEquals("POINT(-22.500000000000004 67.23)", new String(storageFormat)); } @Test diff --git a/core/src/test/java/com/bc/fiduceo/location/PixelGeoCodingPixelLocatorTest.java b/core/src/test/java/com/bc/fiduceo/location/PixelGeoCodingPixelLocatorTest.java new file mode 100644 index 000000000..ef2ebe093 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/location/PixelGeoCodingPixelLocatorTest.java @@ -0,0 +1,64 @@ +package com.bc.fiduceo.location; + +import org.esa.snap.core.dataio.geocoding.GeoChecks; +import org.junit.Before; +import org.junit.Test; +import ucar.ma2.Array; +import ucar.ma2.DataType; + +import java.awt.geom.Point2D; + +import static org.junit.Assert.assertEquals; + +public class PixelGeoCodingPixelLocatorTest { + + private PixelGeoCodingPixelLocator pixelGeoCodingPixelLocator; + + @Before + public void setUp() { + final float[] longitudes = {176.4837f, 176.38458f, 176.28566f, 176.23f, 176.13022f, 176.03065f}; + final float[] latitudes = {-72.1376f, -72.16673f, -72.19598f, -72.17846f, -72.207436f, -72.236534f}; + + final Array lonArray = Array.factory(DataType.FLOAT, new int[]{2, 3}, longitudes); + final Array latArray = Array.factory(DataType.FLOAT, new int[]{2, 3}, latitudes); + + pixelGeoCodingPixelLocator = new PixelGeoCodingPixelLocator(lonArray, latArray, "longitude", "latituide", 35, GeoChecks.NONE); + } + + @Test + public void testGetGeoLocation() { + Point2D geoLocation = pixelGeoCodingPixelLocator.getGeoLocation(0, 0, null); + assertEquals(176.48370361328125, geoLocation.getX(), 1e-8); + assertEquals(-72.13760375976562, geoLocation.getY(), 1e-8); + + geoLocation = pixelGeoCodingPixelLocator.getGeoLocation(1, 1, null); + assertEquals(176.13021850585938, geoLocation.getX(), 1e-8); + assertEquals(-72.20743560791016, geoLocation.getY(), 1e-8); + + geoLocation = pixelGeoCodingPixelLocator.getGeoLocation(-1, 0, null); + assertEquals(Double.NaN, geoLocation.getX(), 1e-8); + assertEquals(Double.NaN, geoLocation.getY(), 1e-8); + + geoLocation = pixelGeoCodingPixelLocator.getGeoLocation(1, 17, null); + assertEquals(Double.NaN, geoLocation.getX(), 1e-8); + assertEquals(Double.NaN, geoLocation.getY(), 1e-8); + } + + @Test + public void testGetPixelLocation() { + Point2D[] pixelLocation = pixelGeoCodingPixelLocator.getPixelLocation(176.48370361328125, -72.13760375976562); + assertEquals(1, pixelLocation.length); + assertEquals(0, pixelLocation[0].getX(), 1e-8); + assertEquals(0, pixelLocation[0].getY(), 1e-8); + + pixelLocation = pixelGeoCodingPixelLocator.getPixelLocation(176.13021850585938, -72.20743560791016); + assertEquals(1, pixelLocation.length); + assertEquals(1, pixelLocation[0].getX(), 1e-8); + assertEquals(1, pixelLocation[0].getY(), 1e-8); + + pixelLocation = pixelGeoCodingPixelLocator.getPixelLocation(22.9, 0.8); + assertEquals(1, pixelLocation.length); + assertEquals(Double.NaN, pixelLocation[0].getX(), 1e-8); + assertEquals(Double.NaN, pixelLocation[0].getY(), 1e-8); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/math/SphericalDistanceTest.java b/core/src/test/java/com/bc/fiduceo/math/SphericalDistanceTest.java index 77f3159cd..2adf130e7 100644 --- a/core/src/test/java/com/bc/fiduceo/math/SphericalDistanceTest.java +++ b/core/src/test/java/com/bc/fiduceo/math/SphericalDistanceTest.java @@ -35,5 +35,14 @@ public void testDistance() { assertEquals(1.0, Math.toDegrees(distanceMeasure.distance(1.0, 0.0)), 1.0e-10); assertEquals(1.0, Math.toDegrees(distanceMeasure.distance(0.0, 1.0)), 1.0e-10); } + + @Test + public void testDistanceHalfDegree() { + final double lat = 60.0; + final DistanceMeasure distanceMeasure = new SphericalDistance(0.0, lat); + + assertEquals(0.0, distanceMeasure.distance(0.0, lat), 0.0); + assertEquals(0.5, Math.toDegrees(distanceMeasure.distance(1.0, lat)), 1e-5); + } } diff --git a/core/src/test/java/com/bc/fiduceo/qc/MmdQCToolTest.java b/core/src/test/java/com/bc/fiduceo/qc/MmdQCToolTest.java index b312488a8..9f932ef24 100644 --- a/core/src/test/java/com/bc/fiduceo/qc/MmdQCToolTest.java +++ b/core/src/test/java/com/bc/fiduceo/qc/MmdQCToolTest.java @@ -7,7 +7,6 @@ import java.io.ByteArrayOutputStream; import static org.junit.Assert.*; -import static org.junit.Assert.assertFalse; public class MmdQCToolTest { @@ -18,19 +17,24 @@ public void testPrintUsageTo() { MmdQCTool.printUsageTo(outputStream); - assertEquals("mmd-qc-tool version 1.5.8" + ls + + assertEquals("mmd-qc-tool version 1.6.2" + ls + ls + - "usage: matchup-tool " + ls + + "usage: mmd-qc-tool " + ls + "Valid options are:" + ls + - " -h,--help Prints the tool usage." + ls + - " -i,--input Defines the MMD input directory." + ls + - " -t,--time Defines matchup time variable name." + ls, outputStream.toString()); + " -h,--help Prints the tool usage." + ls + + " -i,--input Defines the MMD input directory." + ls + + " -lat,--latitude Defines the variable name for the latitude." + ls + + " -lon,--longitude Defines the variable name for the longitude." + ls + + " -o,--outdir Defines the result output directory." + ls + + " -p,--plot Enables plotting the matchup locations onto a global map. Requires 'lon' and 'lat' to be" + ls + + " set." + ls + + " -t,--time Defines matchup time variable name." + ls, outputStream.toString()); } @Test public void testGetOptions() { final Options options = MmdQCTool.getOptions(); - assertEquals(3, options.getOptions().size()); + assertEquals(7, options.getOptions().size()); Option o = options.getOption("h"); assertEquals("help", o.getLongOpt()); @@ -44,11 +48,35 @@ public void testGetOptions() { assertTrue(o.hasArg()); assertTrue(o.isRequired()); + o = options.getOption("o"); + assertEquals("outdir", o.getLongOpt()); + assertEquals("Defines the result output directory.", o.getDescription()); + assertTrue(o.hasArg()); + assertFalse(o.isRequired()); + o = options.getOption("t"); assertEquals("time", o.getLongOpt()); assertEquals("Defines matchup time variable name.", o.getDescription()); assertTrue(o.hasArg()); assertTrue(o.isRequired()); + + o = options.getOption("p"); + assertEquals("plot", o.getLongOpt()); + assertEquals("Enables plotting the matchup locations onto a global map. Requires 'lon' and 'lat' to be set.", o.getDescription()); + assertFalse(o.hasArg()); + assertFalse(o.isRequired()); + + o = options.getOption("lon"); + assertEquals("longitude", o.getLongOpt()); + assertEquals("Defines the variable name for the longitude.", o.getDescription()); + assertTrue(o.hasArg()); + assertFalse(o.isRequired()); + + o = options.getOption("lat"); + assertEquals("latitude", o.getLongOpt()); + assertEquals("Defines the variable name for the latitude.", o.getDescription()); + assertTrue(o.hasArg()); + assertFalse(o.isRequired()); } @Test @@ -83,4 +111,32 @@ public void testWriteReport_one_file_some_matches_one_day() { "Daily distribution:" + ls + "2022-08-12: 5" + ls, outputStream.toString()); } + + @Test + public void testWriteReport_one_file_some_matches_with_errors() { + final String ls = System.lineSeparator(); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + final MatchupAccumulator accumulator = new MatchupAccumulator(); + + accumulator.countFile(); + + accumulator.add(1660262400); + accumulator.add(1660262800); + + final FileMessages fileMessages = new FileMessages(); + fileMessages.add("heffalump_3.nc", "Unable to read metadata"); + fileMessages.add("heffalump_3.nc", "Quirky format"); + + + MmdQCTool.writeReport(outputStream, accumulator, fileMessages); + + assertEquals("Analysed 1 file(s)" + ls + ls + + "1 file(s) with errors" + ls + + "heffalump_3.nc" + ls + + " - Unable to read metadata" + ls + + " - Quirky format" + ls + ls + + "Total number of matchups: 2" + ls + + "Daily distribution:" + ls + + "2022-08-12: 2" + ls, outputStream.toString()); + } } diff --git a/core/src/test/java/com/bc/fiduceo/reader/RawDataReaderTest.java b/core/src/test/java/com/bc/fiduceo/reader/RawDataReaderTest.java index 226bd0a97..a1dafa8ea 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/RawDataReaderTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/RawDataReaderTest.java @@ -23,7 +23,13 @@ import org.junit.Test; +import java.awt.*; +import java.awt.geom.Rectangle2D; + +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.CoreMatchers.*; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; public class RawDataReaderTest { @@ -43,4 +49,32 @@ public void testGetInputDimensions() { assertEquals(RawDataReader.InputDimension.THREE_D_FALSE_DIMENSION, RawDataReader.getInputDimension(3, new int[]{1, 409, 4443})); assertEquals(RawDataReader.InputDimension.THREE_D_FALSE_DIMENSION, RawDataReader.getInputDimension(3, new int[]{1, 10, 10})); } + + @Test + public void testGetInsideWindow() { + // Assumptions: + // Given is a source array with a width of 200 and a height of 100 + // It is to be read from different areas from the array + // Mostly somewhere inside the array. But sometimes also areas that are partially outside. + + Rectangle2D insideWindow; + + // case: inside + insideWindow = RawDataReader.getInsideWindow(50, 40, 4, 3, 200, 100); + assertThat(insideWindow, is(not(nullValue()))); + assertThat(insideWindow, is(instanceOf(Rectangle.class))); + assertThat(insideWindow, is(equalTo(new Rectangle(50, 40, 4, 3)))); + + // case: partly outside top left + insideWindow = RawDataReader.getInsideWindow(-1, -1, 4, 3, 200, 100); + assertThat(insideWindow, is(not(nullValue()))); + assertThat(insideWindow, is(instanceOf(Rectangle.class))); + assertThat(insideWindow, is(equalTo(new Rectangle(0, 0, 3, 2)))); + + // case: partly outside lower right + insideWindow = RawDataReader.getInsideWindow(197, 98, 4, 3, 200, 100); + assertThat(insideWindow, is(not(nullValue()))); + assertThat(insideWindow, is(instanceOf(Rectangle.class))); + assertThat(insideWindow, is(equalTo(new Rectangle(197, 98, 3, 2)))); + } } diff --git a/core/src/test/java/com/bc/fiduceo/reader/RawDataReaderTest_context2D_double.java b/core/src/test/java/com/bc/fiduceo/reader/RawDataReaderTest_context2D_double.java index 377c18bc1..603fa0bf9 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/RawDataReaderTest_context2D_double.java +++ b/core/src/test/java/com/bc/fiduceo/reader/RawDataReaderTest_context2D_double.java @@ -5,9 +5,7 @@ import com.bc.fiduceo.util.NetCDFUtils; import org.junit.Before; import org.junit.Test; -import sun.nio.ch.Net; import ucar.ma2.Array; -import ucar.ma2.DataType; import java.io.IOException; diff --git a/core/src/test/java/com/bc/fiduceo/reader/ReaderContextTest.java b/core/src/test/java/com/bc/fiduceo/reader/ReaderContextTest.java index 965744b7a..953690426 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/ReaderContextTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/ReaderContextTest.java @@ -1,7 +1,6 @@ package com.bc.fiduceo.reader; import com.bc.fiduceo.archive.Archive; -import com.bc.fiduceo.archive.ArchiveConfig; import com.bc.fiduceo.geometry.GeometryFactory; import com.bc.fiduceo.util.TempFileUtils; import org.junit.Before; diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsr/amsr2/AMSR2_Reader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/amsr/amsr2/AMSR2_Reader_IO_Test.java index 53882e633..736e1260b 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/amsr/amsr2/AMSR2_Reader_IO_Test.java +++ b/core/src/test/java/com/bc/fiduceo/reader/amsr/amsr2/AMSR2_Reader_IO_Test.java @@ -423,7 +423,7 @@ public void testGetPixelLocator() throws IOException { Point2D[] pixelLocation = pixelLocator.getPixelLocation(48.574527740478516, 8.604050636291504); assertEquals(1, pixelLocation.length); assertEquals(127.48316325755054, pixelLocation[0].getX(), 1e-8); - assertEquals(1122.479083995203, pixelLocation[0].getY(), 1e-8); + assertEquals(1122.479083995203, pixelLocation[0].getY(), 1e-6); pixelLocation = pixelLocator.getPixelLocation(-4, 1176); assertEquals(0, pixelLocation.length); diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsr/amsre/AMSRE_Reader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/amsr/amsre/AMSRE_Reader_IO_Test.java index 711791d75..3fbeea98f 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/amsr/amsre/AMSRE_Reader_IO_Test.java +++ b/core/src/test/java/com/bc/fiduceo/reader/amsr/amsre/AMSRE_Reader_IO_Test.java @@ -459,7 +459,7 @@ public void testGetPixelLocator() throws IOException { Point2D[] pixelLocation = pixelLocator.getPixelLocation(-51.034195, 52.659184); assertEquals(1, pixelLocation.length); assertEquals(132.51219668358698, pixelLocation[0].getX(), 1e-8); - assertEquals(411.4166814102514, pixelLocation[0].getY(), 1e-8); + assertEquals(411.4166814102514, pixelLocation[0].getY(), 1e-6); pixelLocation = pixelLocator.getPixelLocation(-4, 1176); assertEquals(0, pixelLocation.length); diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_ReaderPluginTest.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_ReaderPluginTest.java new file mode 100644 index 000000000..344937142 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_ReaderPluginTest.java @@ -0,0 +1,40 @@ +package com.bc.fiduceo.reader.amsu_mhs; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.ReaderContext; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; + +public class AMSUA_L1B_ReaderPluginTest { + + private AMSUA_L1B_ReaderPlugin plugin; + + @Before + public void setUp() { + plugin = new AMSUA_L1B_ReaderPlugin(); + } + + @Test + public void testGetSupportedSensorKeys() { + final String[] expected = {"amsua-ma-l1b", "amsua-mb-l1b", "amsua-mc-l1b"}; + + final String[] sensorKeys = plugin.getSupportedSensorKeys(); + assertArrayEquals(expected, sensorKeys); + } + + @Test + public void testGetDataType() { + assertEquals(DataType.POLAR_ORBITING_SATELLITE, plugin.getDataType()); + } + + @Test + public void testCreateReader() { + final Reader reader = plugin.createReader(new ReaderContext()); + assertNotNull(reader); + assertTrue(reader instanceof AMSUA_L1B_Reader); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_ReaderTest.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_ReaderTest.java new file mode 100644 index 000000000..6b0772728 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_ReaderTest.java @@ -0,0 +1,157 @@ +package com.bc.fiduceo.reader.amsu_mhs; + +import com.bc.fiduceo.reader.ReaderContext; +import com.bc.fiduceo.reader.amsu_mhs.nat.GENERIC_RECORD_HEADER; +import com.bc.fiduceo.reader.amsu_mhs.nat.VariableDefinition; +import org.junit.Test; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; + +import java.io.IOException; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertFalse; + +@SuppressWarnings("resource") +public class AMSUA_L1B_ReaderTest { + + @Test + public void testEnsureMdrVersionSupported() { + final byte[] data = {8, 1, 2, 3, 0, 0, 0, 100}; + final GENERIC_RECORD_HEADER recordHeader = GENERIC_RECORD_HEADER.parse(data); + + try { + AMSUA_L1B_Reader.ensureMdrVersionSupported(recordHeader); + } catch (IllegalStateException e) { + fail("no exception expected"); + } + } + + @Test + public void testEnsureMdrVersionSupported_wrongVersion() { + // ----------------------- ! ---------------- + final byte[] data = {8, 1, 3, 3, 0, 0, 0, 100}; + final GENERIC_RECORD_HEADER recordHeader = GENERIC_RECORD_HEADER.parse(data); + + try { + AMSUA_L1B_Reader.ensureMdrVersionSupported(recordHeader); + fail("IllegalStateException expected"); + } catch (IllegalStateException expected) { + } + } + + @Test + public void testEnsureMdrVersionSupported_wrongSubVersion() { + // -------------------------- ! ------------- + final byte[] data = {8, 1, 2, 5, 0, 0, 0, 100}; + final GENERIC_RECORD_HEADER recordHeader = GENERIC_RECORD_HEADER.parse(data); + + try { + AMSUA_L1B_Reader.ensureMdrVersionSupported(recordHeader); + fail("IllegalStateException expected"); + } catch (IllegalStateException expected) { + } + } + + @Test + public void testGetRegEx() { + final AMSUA_L1B_Reader reader = new AMSUA_L1B_Reader(new ReaderContext()); + + final String regEx = reader.getRegEx(); + assertEquals("AMSA_[A-Z0-9x]{3}_1B_M0[123]_[0-9]{14}Z_[0-9]{14}Z_[A-Z0-9x]{1}_[A-Z0-9x]{1}_[0-9]{14}Z\\.nat", regEx); + + final Pattern pattern = Pattern.compile(regEx); + Matcher matcher = pattern.matcher("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + assertTrue(matcher.matches()); + + matcher = pattern.matcher("ABCD_xxx_1C_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + assertFalse(matcher.matches()); + + matcher = pattern.matcher("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + assertFalse(matcher.matches()); + + matcher = pattern.matcher("MHSx_xxx_1B_M04_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + assertFalse(matcher.matches()); + } + + @Test + public void testGetLongitudeVariableName() { + final AMSUA_L1B_Reader reader = new AMSUA_L1B_Reader(new ReaderContext()); + + assertEquals("longitude", reader.getLongitudeVariableName()); + } + + @Test + public void testGetLatitudeVariableName() { + final AMSUA_L1B_Reader reader = new AMSUA_L1B_Reader(new ReaderContext()); + + assertEquals("latitude", reader.getLatitudeVariableName()); + } + + @Test + public void testExtractYearMonthDayFromFilename() { + final AMSUA_L1B_Reader reader = new AMSUA_L1B_Reader(new ReaderContext()); + + assertArrayEquals(new int[]{2016, 1, 1}, reader.extractYearMonthDayFromFilename("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat")); + } + + @Test + public void testExtractCFAttributes() { + final VariableDefinition definition = new VariableDefinition(); + definition.setUnits("m/s"); + definition.setScale_factor(1.8); + definition.setData_type("integer4"); + definition.setFlag_meanings("hans frida bert agathe mümmelmann"); + definition.setFlag_values("1, 2, 4, 8, 16"); + definition.setStandard_name("gamble_man"); + + final List attributes = AMSUA_L1B_Reader.extractCFAttributes(definition); + assertEquals(7, attributes.size()); + + Attribute attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("m/s", attribute.getStringValue()); + + attribute = attributes.get(1); + assertEquals("scale_factor", attribute.getShortName()); + assertEquals(1.8, attribute.getNumericValue()); + + attribute = attributes.get(2); + assertEquals("add_offset", attribute.getShortName()); + assertEquals(0.0, attribute.getNumericValue()); + + attribute = attributes.get(3); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(-2147483648, attribute.getNumericValue()); + + attribute = attributes.get(4); + assertEquals("flag_meanings", attribute.getShortName()); + assertEquals("hans frida bert agathe mümmelmann", attribute.getStringValue()); + + attribute = attributes.get(5); + assertEquals("flag_values", attribute.getShortName()); + Array values = attribute.getValues(); + assertEquals(5, values.getSize()); + + attribute = attributes.get(6); + assertEquals("standard_name", attribute.getShortName()); + assertEquals("gamble_man", attribute.getStringValue()); + } + + @Test + public void testToValuesArray() { + final String valuesString = "1, 2, 4, 8, 16"; + + final Array intArray = AMSUA_L1B_Reader.toValuesArray(valuesString, "integer4"); + assertEquals(DataType.INT, intArray.getDataType()); + assertEquals(5, intArray.getSize()); + assertEquals(1, intArray.getInt(0)); + assertEquals(8, intArray.getInt(3)); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_Reader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_Reader_IO_Test.java new file mode 100644 index 000000000..c665c5480 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUA_L1B_Reader_IO_Test.java @@ -0,0 +1,478 @@ +package com.bc.fiduceo.reader.amsu_mhs; + +import com.bc.fiduceo.IOTestRunner; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.core.NodeType; +import com.bc.fiduceo.geometry.*; +import com.bc.fiduceo.location.PixelLocator; +import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.ReaderContext; +import com.bc.fiduceo.reader.time.TimeLocator; +import com.bc.fiduceo.util.VariableProxy; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.*; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; + +import java.awt.geom.Point2D; +import java.io.File; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +import static com.bc.fiduceo.geometry.GeometryFactory.Type.S2; +import static org.junit.Assert.*; + +@RunWith(IOTestRunner.class) +public class AMSUA_L1B_Reader_IO_Test { + + private AMSUA_L1B_Reader reader; + + @Before + public void setUp() { + final ReaderContext readerContext = new ReaderContext(); + readerContext.setGeometryFactory(new GeometryFactory(S2)); + reader = new AMSUA_L1B_Reader(readerContext); + } + + @Test + public void testReadAcquisitionInfo_MetopA() throws IOException, ParseException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + + try { + reader.open(file); + + final AcquisitionInfo acquisitionInfo = reader.read(); + + final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmssX"); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + final Date expectedStart = sdf.parse("20160101234924Z"); + final Date expectedStop = sdf.parse("20160102013124Z"); + + assertEquals(expectedStart, acquisitionInfo.getSensingStart()); + assertEquals(expectedStop, acquisitionInfo.getSensingStop()); + assertEquals(NodeType.UNDEFINED, acquisitionInfo.getNodeType()); + + final Geometry boundingGeometry = acquisitionInfo.getBoundingGeometry(); + assertTrue(boundingGeometry instanceof MultiPolygon); + final MultiPolygon multiPolygon = (MultiPolygon) boundingGeometry; + final List polygons = multiPolygon.getPolygons(); + assertEquals(2, polygons.size()); + + final Point[] coordinates0 = polygons.get(0).getCoordinates(); + assertEquals(51, coordinates0.length); + assertEquals(-168.0057, coordinates0[0].getLon(), 1e-8); + assertEquals(65.5792, coordinates0[0].getLat(), 1e-8); + assertEquals(169.72060000000005, coordinates0[48].getLon(), 1e-8); + assertEquals(50.1226, coordinates0[48].getLat(), 1e-8); + + Point[] coordinates1 = polygons.get(1).getCoordinates(); + assertEquals(51, coordinates1.length); + assertEquals(-51.4854, coordinates1[0].getLon(), 1e-8); + assertEquals(-73.0652, coordinates1[0].getLat(), 1e-8); + assertEquals(-52.7916, coordinates1[48].getLon(), 1e-8); + assertEquals(-54.673, coordinates1[48].getLat(), 1e-8); + + final TimeAxis[] timeAxes = acquisitionInfo.getTimeAxes(); + assertEquals(2, timeAxes.length); + TimeAxis timeAxis = timeAxes[0]; + Point[] coordinates = timeAxis.getGeometry().getCoordinates(); + assertEquals(21, coordinates.length); + assertEquals(161.1958, coordinates[1].getLon(), 1e-8); + assertEquals(62.6422, coordinates[1].getLat(), 1e-8); + + timeAxis = timeAxes[1]; + coordinates = timeAxis.getGeometry().getCoordinates(); + assertEquals(21, coordinates.length); + assertEquals(-31.1957, coordinates[1].getLon(), 1e-8); + assertEquals(-61.7517, coordinates[1].getLat(), 1e-8); + + Date time = timeAxes[0].getTime(coordinates0[0]); + TestUtil.assertCorrectUTCDate(2016, 1, 1, 23, 49, 24, 0, time); + time = timeAxes[1].getTime(coordinates1[1]); + TestUtil.assertCorrectUTCDate(2016, 1, 2, 0, 40, 24, 0, time); + } finally { + reader.close(); + } + } + + @Test + public void testGetTimeLocator() throws IOException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + + try { + reader.open(file); + + final TimeLocator timeLocator = reader.getTimeLocator(); + long time = timeLocator.getTimeFor(3, 0); + TestUtil.assertCorrectUTCDate(2016, 1, 1, 23, 49, 24, 0, new Date(time)); + + time = timeLocator.getTimeFor(4, 250); + TestUtil.assertCorrectUTCDate(2016, 1, 2, 0, 22, 46, 618, new Date(time)); + + time = timeLocator.getTimeFor(5, 764); + TestUtil.assertCorrectUTCDate(2016, 1, 2, 1, 31, 24, 0, new Date(time)); + } finally { + reader.close(); + } + } + + @Test + public void testGetPixelLocator() throws IOException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + + try { + reader.open(file); + + final PixelLocator pixelLocator = reader.getPixelLocator(); + Point2D geoLocation = pixelLocator.getGeoLocation(0, 0, null); + assertEquals(-168.0057, geoLocation.getX(), 1e-8); + assertEquals(65.5792, geoLocation.getY(), 1e-8); + + geoLocation = pixelLocator.getGeoLocation(5, 108, null); + assertEquals(150.1052, geoLocation.getX(), 1e-8); + assertEquals(20.8303, geoLocation.getY(), 1e-8); + + Point2D[] pixelLocations = pixelLocator.getPixelLocation(-168.0057000002, 65.5792); + assertEquals(1, pixelLocations.length); + assertEquals(0, pixelLocations[0].getX(), 1e-8); + assertEquals(0, pixelLocations[0].getY(), 1e-8); + + pixelLocations = pixelLocator.getPixelLocation(150.1052, 20.8303); + assertEquals(1, pixelLocations.length); + assertEquals(5, pixelLocations[0].getX(), 1e-8); + assertEquals(108, pixelLocations[0].getY(), 1e-8); + } finally { + reader.close(); + } + } + + @Test + public void testGetSubScenePixelLocator() throws IOException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + try { + reader.open(file); + + final PixelLocator pixelLocator = reader.getPixelLocator(); + final PixelLocator subScenePixelLocator = reader.getSubScenePixelLocator(null); + assertSame(pixelLocator, subScenePixelLocator); + } finally { + reader.close(); + } + } + + @Test + public void testGetProductSize() throws IOException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + try { + reader.open(file); + + final Dimension productSize = reader.getProductSize(); + assertEquals(30, productSize.getNx(), 1e-8); + assertEquals(765, productSize.getNy(), 1e-8); + } finally { + reader.close(); + } + } + + @Test + public void testReadRaw() throws IOException, InvalidRangeException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + try { + reader.open(file); + + Array rawData = reader.readRaw(1, 1, new Interval(3, 3), "SCENE_RADIANCE_01"); + Index idx = rawData.getIndex(); + idx.set(0, 0); + assertEquals(11880, rawData.getInt(idx)); + idx.set(0, 1); + assertEquals(12634, rawData.getInt(idx)); + idx.set(0, 2); + assertEquals(12883, rawData.getInt(idx)); + + rawData = reader.readRaw(2, 2, new Interval(3, 3), "solar_zenith_angle"); + idx = rawData.getIndex(); + idx.set(1, 0); + assertEquals(8883, rawData.getInt(idx)); + idx.set(1, 1); + assertEquals(8947, rawData.getInt(idx)); + idx.set(1, 2); + assertEquals(9002, rawData.getInt(idx)); + } finally { + reader.close(); + } + } + + @Test + public void testReadRaw_topBorder() throws IOException, InvalidRangeException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + try { + reader.open(file); + + Array rawData = reader.readRaw(1, 0, new Interval(3, 3), "SCENE_RADIANCE_02"); + Index idx = rawData.getIndex(); + idx.set(0, 0); + assertEquals(-2147483648, rawData.getInt(idx)); + idx.set(1, 1); + assertEquals(21796, rawData.getInt(idx)); + idx.set(2, 2); + assertEquals(21208, rawData.getInt(idx)); + } finally { + reader.close(); + } + } + + @Test + public void testReadRaw_upperRightEdge() throws IOException, InvalidRangeException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + try { + reader.open(file); + + Array rawData = reader.readRaw(29, 0, new Interval(3, 3), "SURFACE_PROPERTIES"); + Index idx = rawData.getIndex(); + idx.set(0, 2); + assertEquals(-32768, rawData.getInt(idx)); + idx.set(1, 2); + assertEquals(-32768, rawData.getInt(idx)); + idx.set(0, 1); + assertEquals(-32768, rawData.getInt(idx)); + idx.set(1, 1); + assertEquals(1, rawData.getInt(idx)); + idx.set(2, 1); + assertEquals(1, rawData.getInt(idx)); + } finally { + reader.close(); + } + } + + @Test + public void testReadScaled() throws IOException, InvalidRangeException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + try { + reader.open(file); + + Array scaledData = reader.readScaled(3, 3, new Interval(3, 3), "SCENE_RADIANCE_03"); + Index idx = scaledData.getIndex(); + idx.set(0, 0); + assertEquals(0.005504000000655651, scaledData.getFloat(idx), 1e-8); + idx.set(0, 1); + assertEquals(0.005510400049388409, scaledData.getFloat(idx), 1e-8); + idx.set(0, 2); + assertEquals(0.005386400036513805, scaledData.getFloat(idx), 1e-8); + + scaledData = reader.readScaled(4, 4, new Interval(3, 3), "satellite_azimuth_angle"); + idx = scaledData.getIndex(); + idx.set(1, 0); + assertEquals(-50.310001373291016, scaledData.getFloat(idx), 1e-8); + idx.set(1, 1); + assertEquals(-51.689998626708984, scaledData.getFloat(idx), 1e-8); + idx.set(1, 2); + assertEquals(-52.97999954223633, scaledData.getFloat(idx), 1e-8); + + // and check for a variable without scale factor tb 2025-09-17 + scaledData = reader.readScaled(4, 4, new Interval(3, 3), "TERRAIN_ELEVATION"); + idx = scaledData.getIndex(); + idx.set(1, 0); + assertEquals(244, scaledData.getFloat(idx), 1e-8); + idx.set(1, 1); + assertEquals(305, scaledData.getFloat(idx), 1e-8); + idx.set(1, 2); + assertEquals(457, scaledData.getFloat(idx), 1e-8); + } finally { + reader.close(); + } + } + + @Test + public void testReadScaled_rightBorder() throws IOException, InvalidRangeException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + try { + reader.open(file); + + Array scaledData = reader.readScaled(29, 5, new Interval(3, 3), "SCENE_RADIANCE_04"); + Index idx = scaledData.getIndex(); + idx.set(0, 0); + assertEquals(0.006011600140482187, scaledData.getFloat(idx), 1e-8); + idx.set(1, 1); + assertEquals(0.005992699880152941, scaledData.getFloat(idx), 1e-8); + idx.set(2, 2); + assertEquals(-214.7483673095703, scaledData.getFloat(idx), 1e-8); + } finally { + reader.close(); + } + } + + @Test + public void testReadScaled_lowerRightEdge() throws IOException, InvalidRangeException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + try { + reader.open(file); + + Array scaledData = reader.readScaled(29, 764, new Interval(3, 3), "satellite_zenith_angle"); + Index idx = scaledData.getIndex(); + idx.set(0, 0); + assertEquals(53.040000915527344, scaledData.getFloat(idx), 1e-8); + idx.set(1, 1); + assertEquals(57.59000015258789, scaledData.getFloat(idx), 1e-8); + idx.set(2, 2); + assertEquals(-327.67999267578125, scaledData.getFloat(idx), 1e-8); + } finally { + reader.close(); + } + } + + + @Test + public void testGetVariables() throws InvalidRangeException, IOException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + try { + reader.open(file); + + final List variables = reader.getVariables(); + assertEquals(24, variables.size()); + + VariableProxy variable = (VariableProxy) variables.get(0); + assertEquals("solar_zenith_angle", variable.getFullName()); + assertEquals(DataType.SHORT, variable.getDataType()); + List attributes = variable.getAttributes(); + Attribute attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("degree", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("scale_factor", attribute.getShortName()); + assertEquals(0.01, attribute.getNumericValue()); + attribute = attributes.get(2); + assertEquals("add_offset", attribute.getShortName()); + assertEquals(0.0, attribute.getNumericValue()); + attribute = attributes.get(3); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(-32768, attribute.getNumericValue().intValue()); + attribute = attributes.get(4); + assertEquals("standard_name", attribute.getShortName()); + assertEquals("solar_zenith_angle", attribute.getStringValue()); + + variable = (VariableProxy) variables.get(5); + assertEquals("longitude", variable.getFullName()); + assertEquals(DataType.INT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("degree", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("scale_factor", attribute.getShortName()); + assertEquals(1.0E-4, attribute.getNumericValue()); + attribute = attributes.get(3); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(-2147483648, attribute.getNumericValue()); + + variable = (VariableProxy) variables.get(6); + assertEquals("SURFACE_PROPERTIES", variable.getFullName()); + assertEquals(DataType.SHORT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(1); + assertEquals("flag_meanings", attribute.getShortName()); + assertEquals("water mixed_coast land", attribute.getStringValue()); + attribute = attributes.get(2); + assertEquals("flag_values", attribute.getShortName()); + assertEquals(0, attribute.getValues().getShort(0)); + assertEquals(1, attribute.getValues().getShort(1)); + assertEquals(2, attribute.getValues().getShort(2)); + + variable = (VariableProxy) variables.get(10); + assertEquals("SCENE_RADIANCE_07", variable.getFullName()); + assertEquals(DataType.INT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("mW/m2/sr/cm-1", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("scale_factor", attribute.getShortName()); + assertEquals(1.0E-7, attribute.getNumericValue()); + attribute = attributes.get(3); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(-2147483648, attribute.getNumericValue()); + attribute = attributes.get(4); + assertEquals("standard_name", attribute.getShortName()); + assertEquals("toa_radiance", attribute.getStringValue()); + + variable = (VariableProxy) variables.get(15); + assertEquals("SCENE_RADIANCE_14", variable.getFullName()); + assertEquals(DataType.INT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("mW/m2/sr/cm-1", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("scale_factor", attribute.getShortName()); + assertEquals(1.0E-7, attribute.getNumericValue()); + attribute = attributes.get(3); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(-2147483648, attribute.getNumericValue()); + + variable = (VariableProxy) variables.get(20); + assertEquals("SCENE_RADIANCE_02", variable.getFullName()); + assertEquals(DataType.INT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("mW/m2/sr/cm-1", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("scale_factor", attribute.getShortName()); + assertEquals(1.0E-7, attribute.getNumericValue()); + attribute = attributes.get(2); + assertEquals("add_offset", attribute.getShortName()); + assertEquals(0.0, attribute.getNumericValue()); + attribute = attributes.get(3); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(-2147483648, attribute.getNumericValue()); + + variable = (VariableProxy) variables.get(23); + assertEquals("SCENE_RADIANCE_11", variable.getFullName()); + assertEquals(DataType.INT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("mW/m2/sr/cm-1", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("scale_factor", attribute.getShortName()); + assertEquals(1.0E-7, attribute.getNumericValue()); + attribute = attributes.get(3); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(-2147483648, attribute.getNumericValue()); + attribute = attributes.get(4); + assertEquals("standard_name", attribute.getShortName()); + assertEquals("toa_radiance", attribute.getStringValue()); + } finally { + reader.close(); + } + } + + @Test + public void testReadAcquisitionTime() throws InvalidRangeException, IOException { + final File file = createAmsuaMetopAPath("AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"); + try { + reader.open(file); + + ArrayInt.D2 timeArray = reader.readAcquisitionTime(14, 189, new Interval(3, 5)); + assertEquals(1451693662, timeArray.getInt(0)); + assertEquals(1451693662, timeArray.getInt(1)); + + assertEquals(1451693670, timeArray.getInt(3)); + assertEquals(1451693670, timeArray.getInt(4)); + + } finally { + reader.close(); + } + } + private File createAmsuaMetopAPath(String fileName) throws IOException { + final String testFilePath = TestUtil.assembleFileSystemPath(new String[]{"amsua-ma-l1b", "v8A", "2016", "01", "01", fileName}, false); + return TestUtil.getTestDataFileAsserted(testFilePath); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUB_MHS_L1C_ReaderPluginTest.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUB_MHS_L1C_ReaderPluginTest.java index 95553944a..2ed549d47 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUB_MHS_L1C_ReaderPluginTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUB_MHS_L1C_ReaderPluginTest.java @@ -54,7 +54,7 @@ public void setUp() { } @Test - public void testGetSupportedSensorKey() { + public void testGetSupportedSensorKeys() { final String[] expected = {"amsub-n15", "amsub-n16", "amsub-n17", "mhs-n18", "mhs-n19", "mhs-ma", "mhs-mb"}; final String[] sensorKeys = plugin.getSupportedSensorKeys(); diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUB_MHS_L1C_Reader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUB_MHS_L1C_Reader_IO_Test.java index 296cffe9a..fe4cf43dd 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUB_MHS_L1C_Reader_IO_Test.java +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/AMSUB_MHS_L1C_Reader_IO_Test.java @@ -413,7 +413,7 @@ public void testGetPixelLocator_AMSUB_NOAA15() throws IOException { pixelLocation = pixelLocator.getPixelLocation(-9.3541, 46.5467); assertEquals(1, pixelLocation.length); - assertEquals(20.549521097468954, pixelLocation[0].getX(), 1e-8); // original 20.5 + assertEquals(20.549521097468954, pixelLocation[0].getX(), 1e-6); // original 20.5 assertEquals(484.47231359716176, pixelLocation[0].getY(), 1e-8); // original 484.5 //------------------------------------------------------------- @@ -433,7 +433,7 @@ public void testGetPixelLocator_AMSUB_NOAA15() throws IOException { pixelLocation = pixelLocator.getPixelLocation(-35.8302, -37.3806); assertEquals(1, pixelLocation.length); - assertEquals(40.47791010604271, pixelLocation[0].getX(), 1e-8); // original 40.5 + assertEquals(40.47791010604271, pixelLocation[0].getX(), 1e-6); // original 40.5 assertEquals(1024.3933174972674, pixelLocation[0].getY(), 1e-8); // original 1024.5 //------------------------------------------------------------- @@ -453,8 +453,8 @@ public void testGetPixelLocator_AMSUB_NOAA15() throws IOException { pixelLocation = pixelLocator.getPixelLocation(168.4784, -64.352); assertEquals(1, pixelLocation.length); // is part of self intersecting area - assertEquals(60.49976819818859, pixelLocation[0].getX(), 1e-8); // original 60.5 - assertEquals(1504.4747569343847, pixelLocation[0].getY(), 1e-8); // original 1504.5 + assertEquals(60.49976819818859, pixelLocation[0].getX(), 1e-6); // original 60.5 + assertEquals(1504.4747569343847, pixelLocation[0].getY(), 1e-6); // original 1504.5 } finally { reader.close(); @@ -481,7 +481,7 @@ public void testSubScenePixelLocator_MHS_NOAA18() throws IOException { Point2D[] pixelLocation = subScenePixelLocator.getPixelLocation(-169.9742, -55.0959); assertEquals(1, pixelLocation.length); assertEquals(18.4837931639385, pixelLocation[0].getX(), 1e-8); // original 18.5 - assertEquals(1061.542564721166, pixelLocation[0].getY(), 1e-8); // original 1061.5 + assertEquals(1061.542564721166, pixelLocation[0].getY(), 1e-6); // original 1061.5 } finally { reader.close(); diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_ReaderPluginTest.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_ReaderPluginTest.java new file mode 100644 index 000000000..a15ccd842 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_ReaderPluginTest.java @@ -0,0 +1,40 @@ +package com.bc.fiduceo.reader.amsu_mhs; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.ReaderContext; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + + +public class MHS_L1B_ReaderPluginTest { + + private MHS_L1B_ReaderPlugin plugin; + + @Before + public void setUp() throws Exception { + plugin = new MHS_L1B_ReaderPlugin(); + } + + @Test + public void testGetSupportedSensorKeys() { + final String[] expected = {"mhs-ma-l1b", "mhs-mb-l1b", "mhs-mc-l1b"}; + + final String[] sensorKeys = plugin.getSupportedSensorKeys(); + assertArrayEquals(expected, sensorKeys); + } + + @Test + public void testGetDataType() { + assertEquals(DataType.POLAR_ORBITING_SATELLITE, plugin.getDataType()); + } + + @Test + public void testCreateReader() { + final Reader reader = plugin.createReader(new ReaderContext()); + assertNotNull(reader); + assertTrue(reader instanceof MHS_L1B_Reader); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_ReaderTest.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_ReaderTest.java new file mode 100644 index 000000000..d3b5b34c0 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_ReaderTest.java @@ -0,0 +1,54 @@ +package com.bc.fiduceo.reader.amsu_mhs; + +import com.bc.fiduceo.reader.ReaderContext; +import org.junit.Before; +import org.junit.Test; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.*; + +public class MHS_L1B_ReaderTest { + + private MHS_L1B_Reader reader; + + @Before + public void setUp() { + reader = new MHS_L1B_Reader(new ReaderContext()); + } + + @Test + public void testGetRegEx() { + final String regEx = reader.getRegEx(); + assertEquals("MHSx_[A-Z0-9x]{3}_1B_M0[123]_[0-9]{14}Z_[0-9]{14}Z_[A-Z0-9x]{1}_[A-Z0-9x]{1}_[0-9]{14}Z\\.nat", regEx); + + final Pattern pattern = Pattern.compile(regEx); + Matcher matcher = pattern.matcher("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + assertTrue(matcher.matches()); + + matcher = pattern.matcher("ABCD_xxx_1C_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + assertFalse(matcher.matches()); + + matcher = pattern.matcher("MHSx_xxx_1C_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + assertFalse(matcher.matches()); + + matcher = pattern.matcher("MHSx_xxx_1B_M04_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + assertFalse(matcher.matches()); + } + + @Test + public void testGetLongitudeVariableName() { + assertEquals("longitude", reader.getLongitudeVariableName()); + } + + @Test + public void testGetLatitudeVariableName() { + assertEquals("latitude", reader.getLatitudeVariableName()); + } + + @Test + public void testExtractYearMonthDayFromFilename() { + assertArrayEquals(new int[]{2025, 8, 20}, reader.extractYearMonthDayFromFilename("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat")); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_Reader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_Reader_IO_Test.java new file mode 100644 index 000000000..43c7873bf --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/MHS_L1B_Reader_IO_Test.java @@ -0,0 +1,518 @@ +package com.bc.fiduceo.reader.amsu_mhs; + +import com.bc.fiduceo.IOTestRunner; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.core.NodeType; +import com.bc.fiduceo.geometry.*; +import com.bc.fiduceo.location.PixelLocator; +import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.ReaderContext; +import com.bc.fiduceo.reader.time.TimeLocator; +import com.bc.fiduceo.util.VariableProxy; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.*; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; + +import java.awt.geom.Point2D; +import java.io.File; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +import static org.junit.Assert.*; + +@RunWith(IOTestRunner.class) +public class MHS_L1B_Reader_IO_Test { + + private MHS_L1B_Reader reader; + + @Before + public void setUp() { + final ReaderContext readerContext = new ReaderContext(); + readerContext.setGeometryFactory(new GeometryFactory(GeometryFactory.Type.S2)); + this.reader = new MHS_L1B_Reader(readerContext); + } + + @Test + public void testReadAcquisitionInfo_MetopC() throws IOException, ParseException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + + try { + this.reader.open(file); + final AcquisitionInfo acquisitionInfo = this.reader.read(); + + final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmssX"); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + final Date expectedStart = sdf.parse("20250820060350Z"); + final Date expectedStop = sdf.parse("20250820074550Z"); + + final Geometry boundingGeometry = acquisitionInfo.getBoundingGeometry(); + + assertEquals(expectedStart, acquisitionInfo.getSensingStart()); + assertEquals(expectedStop, acquisitionInfo.getSensingStop()); + assertEquals(NodeType.UNDEFINED, acquisitionInfo.getNodeType()); + + assertNotNull(boundingGeometry); + assertTrue(boundingGeometry instanceof MultiPolygon); + final MultiPolygon multiPolygon = (MultiPolygon) boundingGeometry; + final List polygons = multiPolygon.getPolygons(); + assertEquals(2, polygons.size()); + + Point[] coordinates0 = polygons.get(0).getCoordinates(); + assertEquals(135, coordinates0.length); + assertEquals(84.0214, coordinates0[0].getLon(), 1e-8); + assertEquals(12.696399999999999, coordinates0[48].getLon(), 1e-8); + assertEquals(56.086600000000004, coordinates0[0].getLat(), 1e-8); + assertEquals(-54.333, coordinates0[48].getLat(), 1e-8); + + Point[] coordinates1 = polygons.get(1).getCoordinates(); + assertEquals(135, coordinates1.length); + assertEquals(-145.5978, coordinates1[0].getLon(), 1e-8); + assertEquals(-145.0478, coordinates1[48].getLon(), 1e-8); + assertEquals(-61.4848, coordinates1[0].getLat(), 1e-8); + assertEquals(61.40250000000001, coordinates1[48].getLat(), 1e-8); + + final TimeAxis[] timeAxes = acquisitionInfo.getTimeAxes(); + assertEquals(2, timeAxes.length); + TimeAxis timeAxis = timeAxes[0]; + Point[] coordinates = timeAxis.getGeometry().getCoordinates(); + assertEquals(59, coordinates.length); + assertEquals(65.3291, coordinates[1].getLon(), 1e-8); + assertEquals(57.4655, coordinates[1].getLat(), 1e-8); + + timeAxis = timeAxes[1]; + coordinates = timeAxis.getGeometry().getCoordinates(); + assertEquals(59, coordinates.length); + assertEquals(-127.6211, coordinates[1].getLon(), 1e-8); + assertEquals(-56.5496, coordinates[1].getLat(), 1e-8); + + assertEquals(2, timeAxes.length); + Date time = timeAxes[0].getTime(coordinates0[0]); + TestUtil.assertCorrectUTCDate(2025, 8, 20, 6, 3, 50, 0, time); + time = timeAxes[1].getTime(coordinates1[0]); + TestUtil.assertCorrectUTCDate(2025, 8, 20, 6, 54, 50, 0, time); + + } finally { + this.reader.close(); + } + } + + @Test + public void testGetTimeLocator() throws IOException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + + try { + reader.open(file); + + final TimeLocator timeLocator = reader.getTimeLocator(); + long time = timeLocator.getTimeFor(12, 0); + TestUtil.assertCorrectUTCDate(2025, 8, 20, 6, 3, 50, 0, new Date(time)); + + time = timeLocator.getTimeFor(12, 250); + TestUtil.assertCorrectUTCDate(2025, 8, 20, 6, 14, 56, 957, new Date(time)); + + time = timeLocator.getTimeFor(12, 2294); + TestUtil.assertCorrectUTCDate(2025, 8, 20, 7, 45, 50, 0, new Date(time)); + } finally { + reader.close(); + } + } + + @Test + public void testGetPixelLocator() throws IOException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + + try { + reader.open(file); + + final PixelLocator pixelLocator = reader.getPixelLocator(); + Point2D geoLocation = pixelLocator.getGeoLocation(0, 0, null); + assertEquals(84.0214, geoLocation.getX(), 1e-8); + assertEquals(56.0866, geoLocation.getY(), 1e-8); + + geoLocation = pixelLocator.getGeoLocation(6, 109, null); + assertEquals(68.5426, geoLocation.getX(), 1e-8); + assertEquals(41.5551, geoLocation.getY(), 1e-8); + + Point2D[] pixelLocations = pixelLocator.getPixelLocation(84.0214, 56.0866); + assertEquals(1, pixelLocations.length); + assertEquals(0, pixelLocations[0].getX(), 1e-8); + assertEquals(0, pixelLocations[0].getY(), 1e-8); + + pixelLocations = pixelLocator.getPixelLocation(68.5426, 41.5551); + assertEquals(1, pixelLocations.length); + assertEquals(6, pixelLocations[0].getX(), 1e-8); + assertEquals(109, pixelLocations[0].getY(), 1e-8); + } finally { + reader.close(); + } + } + + @Test + public void testGetSubScenePixelLocator() throws IOException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + try { + reader.open(file); + + final PixelLocator pixelLocator = reader.getPixelLocator(); + final PixelLocator subScenePixelLocator = reader.getSubScenePixelLocator(null); + assertSame(pixelLocator, subScenePixelLocator); + } finally { + reader.close(); + } + } + + @Test + public void testReadRaw() throws IOException, InvalidRangeException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + try { + reader.open(file); + + Array rawData = reader.readRaw(10, 48, new Interval(3, 3), "SCENE_RADIANCES_04"); + Index idx = rawData.getIndex(); + idx.set(0, 0); + assertEquals(777404.0, rawData.getFloat(idx), 1e-8); + idx.set(0, 1); + assertEquals(775187.0, rawData.getFloat(idx), 1e-8); + idx.set(0, 2); + assertEquals(776269.0, rawData.getFloat(idx), 1e-8); + + rawData = reader.readRaw(10, 48, new Interval(3, 3), "latitude"); + idx = rawData.getIndex(); + idx.set(0, 0); + assertEquals(510754.0, rawData.getFloat(idx), 1e-8); + idx.set(0, 1); + assertEquals(511890.0, rawData.getFloat(idx), 1e-8); + idx.set(0, 2); + assertEquals(512965.0, rawData.getFloat(idx), 1e-8); + + rawData = reader.readRaw(10, 48, new Interval(3, 3), "solar_azimuth_angle"); + idx = rawData.getIndex(); + idx.set(0, 0); + assertEquals(15514.0, rawData.getFloat(idx), 1e-8); + idx.set(0, 1); + assertEquals(15463.0, rawData.getFloat(idx), 1e-8); + idx.set(0, 2); + assertEquals(15413.0, rawData.getFloat(idx), 1e-8); + + rawData = reader.readRaw(78, 558, new Interval(3, 3), "SURFACE_PROPERTIES"); + idx = rawData.getIndex(); + idx.set(1, 0); + assertEquals(0, rawData.getFloat(idx), 1e-8); + idx.set(1, 1); + assertEquals(1, rawData.getFloat(idx), 1e-8); + idx.set(1, 2); + assertEquals(1, rawData.getFloat(idx), 1e-8); + + } finally { + reader.close(); + } + } + + @Test + public void testReadRaw_rightBorder() throws IOException, InvalidRangeException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + try { + reader.open(file); + + Array rawData = reader.readRaw(89, 89, new Interval(3, 3), "SCENE_RADIANCES_04"); + Index idx = rawData.getIndex(); + idx.set(0, 0); + assertEquals(789870.0, rawData.getFloat(idx), 1e-8); + idx.set(1, 1); + assertEquals(790777.0, rawData.getFloat(idx), 1e-8); + idx.set(2, 2); + assertEquals(-2.147483648E9, rawData.getFloat(idx), 1e-8); + + } finally { + reader.close(); + } + } + + @Test + public void testReadRaw_upperRightEdge() throws IOException, InvalidRangeException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + try { + reader.open(file); + + Array rawData = reader.readRaw(89, 2282, new Interval(3, 3), "SCENE_RADIANCES_04"); + Index idx = rawData.getIndex(); + idx.set(0, 0); + assertEquals(765371.0, rawData.getFloat(idx), 1e-8); + idx.set(1, 1); + assertEquals(760786.0, rawData.getFloat(idx), 1e-8); + idx.set(2, 2); + assertEquals(-2.147483648E9, rawData.getFloat(idx), 1e-8); + + } finally { + reader.close(); + } + } + + @Test + public void testReadScaled() throws IOException, InvalidRangeException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + try { + reader.open(file); + + Array scaledData = reader.readScaled(10, 48, new Interval(3, 3), "SCENE_RADIANCES_04"); + Index idx = scaledData.getIndex(); + idx.set(0, 0); + assertEquals(0.07774040102958679, scaledData.getFloat(idx), 1e-8); + idx.set(0, 1); + assertEquals(0.07751870155334473, scaledData.getFloat(idx), 1e-8); + idx.set(0, 2); + assertEquals(0.07762689888477325, scaledData.getFloat(idx), 1e-8); + + scaledData = reader.readScaled(10, 48, new Interval(3, 3), "latitude"); + idx = scaledData.getIndex(); + idx.set(0, 0); + assertEquals(51.07540130, scaledData.getFloat(idx), 1e-8); + idx.set(0, 1); + assertEquals(51.18899917, scaledData.getFloat(idx), 1e-8); + idx.set(0, 2); + assertEquals(51.29650115, scaledData.getFloat(idx), 1e-8); + + scaledData = reader.readScaled(10, 48, new Interval(3, 3), "solar_azimuth_angle"); + idx = scaledData.getIndex(); + idx.set(0, 0); + assertEquals(155.13999938, scaledData.getFloat(idx), 1e-8); + idx.set(0, 1); + assertEquals(154.63000488, scaledData.getFloat(idx), 1e-8); + idx.set(0, 2); + assertEquals(154.13000488, scaledData.getFloat(idx), 1e-8); + + scaledData = reader.readScaled(78, 558, new Interval(3, 3), "SURFACE_PROPERTIES"); + idx = scaledData.getIndex(); + idx.set(1, 0); + assertEquals(0, scaledData.getFloat(idx), 1e-8); + idx.set(1, 1); + assertEquals(1, scaledData.getFloat(idx), 1e-8); + idx.set(1, 2); + assertEquals(1, scaledData.getFloat(idx), 1e-8); + } finally { + reader.close(); + } + } + + @Test + public void testReadScaled_rightBorder() throws IOException, InvalidRangeException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + try { + reader.open(file); + + Array scaledData = reader.readScaled(89, 89, new Interval(3, 3), "SCENE_RADIANCES_04"); + Index idx = scaledData.getIndex(); + idx.set(0, 0); + assertEquals(0.0789870023727417, scaledData.getFloat(idx), 1e-8); + idx.set(1, 1); + assertEquals(0.07907769829034805, scaledData.getFloat(idx), 1e-8); + idx.set(2, 2); + assertEquals(-214.7483673095703, scaledData.getFloat(idx), 1e-8); + + } finally { + reader.close(); + } + } + + @Test + public void testReadScaled_upperRightEdge() throws IOException, InvalidRangeException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + try { + reader.open(file); + + Array scaledData = reader.readScaled(89, 2282, new Interval(3, 3), "SCENE_RADIANCES_04"); + Index idx = scaledData.getIndex(); + idx.set(0, 0); + assertEquals(0.0765371024608612, scaledData.getFloat(idx), 1e-8); + idx.set(1, 1); + assertEquals(0.07607860118150711, scaledData.getFloat(idx), 1e-8); + idx.set(2, 2); + assertEquals(-214.7483673095703, scaledData.getFloat(idx), 1e-8); + + } finally { + reader.close(); + } + } + + @Test + public void testGetProductSize() throws IOException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + try { + reader.open(file); + + final Dimension productSize = reader.getProductSize(); + assertEquals(90, productSize.getNx(), 1e-8); + assertEquals(2295, productSize.getNy(), 1e-8); + } finally { + reader.close(); + } + } + + @Test + public void testGetVariables() throws IOException, InvalidRangeException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + try { + reader.open(file); + + final List variables = reader.getVariables(); + assertEquals(15, variables.size()); + + VariableProxy variable = (VariableProxy) variables.get(0); + assertEquals("FOV_DATA_QUALITY", variable.getFullName()); + assertEquals(DataType.INT, variable.getDataType()); + List attributes = variable.getAttributes(); + Attribute attribute = attributes.get(0); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(-2147483648, attribute.getNumericValue()); + attribute = attributes.get(1); + assertEquals("standard_name", attribute.getShortName()); + assertEquals("quality_flag", attribute.getStringValue()); + + variable = (VariableProxy) variables.get(1); + assertEquals("solar_zenith_angle", variable.getFullName()); + assertEquals(DataType.SHORT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("degree", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("scale_factor", attribute.getShortName()); + assertEquals(.01, attribute.getNumericValue()); + attribute = attributes.get(2); + assertEquals("add_offset", attribute.getShortName()); + assertEquals(0.0, attribute.getNumericValue()); + attribute = attributes.get(3); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(Short.MIN_VALUE, attribute.getNumericValue()); + attribute = attributes.get(4); + assertEquals("standard_name", attribute.getShortName()); + assertEquals("solar_zenith_angle", attribute.getStringValue()); + + variable = (VariableProxy) variables.get(5); + assertEquals("latitude", variable.getFullName()); + assertEquals(DataType.INT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("degree", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("scale_factor", attribute.getShortName()); + assertEquals(1.0E-4, attribute.getNumericValue()); + attribute = attributes.get(3); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(-2147483648, attribute.getNumericValue()); + + variable = (VariableProxy) variables.get(7); + assertEquals("SURFACE_PROPERTIES", variable.getFullName()); + assertEquals(DataType.BYTE, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(1); + assertEquals("flag_meanings", attribute.getShortName()); + assertEquals("water mixed_coast land", attribute.getStringValue()); + attribute = attributes.get(2); + assertEquals("flag_values", attribute.getShortName()); + assertEquals(0, attribute.getValues().getShort(0)); + assertEquals(1, attribute.getValues().getShort(1)); + assertEquals(2, attribute.getValues().getShort(2)); + + variable = (VariableProxy) variables.get(8); + assertEquals("TERRAIN_ELEVATION", variable.getFullName()); + assertEquals(DataType.SHORT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("m", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(Short.MIN_VALUE, attribute.getNumericValue()); + attribute = attributes.get(2); + assertEquals("standard_name", attribute.getShortName()); + assertEquals("height_above_mean_sea_level", attribute.getStringValue()); + + variable = (VariableProxy) variables.get(9); + assertEquals("TIME_ATTITUDE", variable.getFullName()); + assertEquals(DataType.INT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("s", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(4294967295L, attribute.getNumericValue()); + attribute = attributes.get(2); + assertEquals("standard_name", attribute.getShortName()); + assertEquals("time", attribute.getStringValue()); + + variable = (VariableProxy) variables.get(10); + assertEquals("SCENE_RADIANCES_02", variable.getFullName()); + assertEquals(DataType.INT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("mW/m2/sr/cm-1", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("scale_factor", attribute.getShortName()); + assertEquals(1.0E-7, attribute.getNumericValue()); + attribute = attributes.get(3); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(-2147483648, attribute.getNumericValue()); + attribute = attributes.get(4); + assertEquals("standard_name", attribute.getShortName()); + assertEquals("toa_radiance", attribute.getStringValue()); + + variable = (VariableProxy) variables.get(13); + assertEquals("SCENE_RADIANCES_05", variable.getFullName()); + assertEquals(DataType.INT, variable.getDataType()); + attributes = variable.getAttributes(); + attribute = attributes.get(0); + assertEquals("units", attribute.getShortName()); + assertEquals("mW/m2/sr/cm-1", attribute.getStringValue()); + attribute = attributes.get(1); + assertEquals("scale_factor", attribute.getShortName()); + assertEquals(1.0E-7, attribute.getNumericValue()); + attribute = attributes.get(3); + assertEquals("_FillValue", attribute.getShortName()); + assertEquals(-2147483648, attribute.getNumericValue()); + attribute = attributes.get(4); + assertEquals("standard_name", attribute.getShortName()); + assertEquals("toa_radiance", attribute.getStringValue()); + + } finally { + reader.close(); + } + } + + @Test + public void getReadAcquisitionTime() throws InvalidRangeException, IOException { + final File file = createMhsMetopCPath("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"); + try { + reader.open(file); + final int width = 3; + final int height = 5; + + ArrayInt.D2 timeArray = reader.readAcquisitionTime(56, 2100, new Interval(width, height)); + assertEquals(width * height, timeArray.getSize()); + assertEquals(1755675427, timeArray.getInt(0)); + assertEquals(1755675427, timeArray.getInt(1)); + + assertEquals(1755675430, timeArray.getInt(3)); + assertEquals(1755675430, timeArray.getInt(4)); + } finally { + reader.close(); + } + } + + private File createMhsMetopCPath(String fileName) throws IOException { + final String testFilePath = TestUtil.assembleFileSystemPath(new String[]{"mhs-mc-l1b", "v10", "2025", "08", "20", fileName}, false); + return TestUtil.getTestDataFileAsserted(testFilePath); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/DATA_LAYOUT_Test.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/DATA_LAYOUT_Test.java new file mode 100644 index 000000000..514e69646 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/DATA_LAYOUT_Test.java @@ -0,0 +1,30 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class DATA_LAYOUT_Test { + + @Test + public void testFromString() { + assertEquals(DATA_LAYOUT.ARRAY, DATA_LAYOUT.fromString("ARRAY")); + assertEquals(DATA_LAYOUT.VECTOR, DATA_LAYOUT.fromString("VECTOR")); + } + + @Test + public void testFromString_invalidArgument() { + try { + DATA_LAYOUT.fromString("HOLY_GRAIL"); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testToString() { + assertEquals("ARRAY", DATA_LAYOUT.toString(DATA_LAYOUT.ARRAY)); + assertEquals("VECTOR", DATA_LAYOUT.toString(DATA_LAYOUT.VECTOR)); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsReaderUtilsTest.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsReaderUtilsTest.java new file mode 100644 index 000000000..3455d6fd9 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsReaderUtilsTest.java @@ -0,0 +1,224 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import com.sun.jna.platform.win32.WinDef; +import org.esa.snap.core.datamodel.ProductData; +import org.junit.Test; +import ucar.ma2.*; + +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import static org.junit.Assert.*; + +public class EpsReaderUtilsTest { + + @Test + public void testReadInt32() { + ByteBuffer buffer = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN); + buffer.putInt(0, 123456789); + assertEquals(123456789, EpsReaderUtils.readInt32(buffer, 0)); + } + + @Test + public void testReadUInt32() { + ByteBuffer buffer = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN); + buffer.putInt(0, -1); + assertEquals(4294967295L, EpsReaderUtils.readUInt32(buffer, 0)); + } + + @Test + public void testReadInt16() { + ByteBuffer buffer = ByteBuffer.allocate(2).order(ByteOrder.BIG_ENDIAN); + buffer.putShort(0, (short) -12345); + assertEquals(-12345, EpsReaderUtils.readInt16(buffer, 0)); + } + + @Test + public void testReadUInt16() { + ByteBuffer buffer = ByteBuffer.allocate(2).order(ByteOrder.BIG_ENDIAN); + buffer.putShort(0, (short) 0xFFFF); + assertEquals(65535, EpsReaderUtils.readUInt16(buffer, 0)); + } + + @Test + public void testReadInt8() { + ByteBuffer buffer = ByteBuffer.allocate(1); + buffer.put(0, (byte) -100); + assertEquals(-100, EpsReaderUtils.readInt8(buffer, 0)); + } + + @Test + public void testReadUInt8() { + ByteBuffer buffer = ByteBuffer.allocate(1); + buffer.put(0, (byte) 0xFF); + assertEquals(255, EpsReaderUtils.readUInt8(buffer, 0)); + } + + @Test + public void testReadInt64() { + ByteBuffer buffer = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN); + buffer.putLong(0, 9876543210L); + assertEquals(9876543210L, EpsReaderUtils.readInt64(buffer, 0)); + } + + @Test + public void testReadUInt64() { + ByteBuffer buffer = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN); + buffer.putLong(0, -1L); + BigInteger expected = new BigInteger("18446744073709551615"); + assertEquals(expected, EpsReaderUtils.readUInt64(buffer, 0)); + } + + @Test + public void testMapToProductDataType() { + int type_byte = EpsReaderUtils.mapToProductData("byte"); + int type_boolean = EpsReaderUtils.mapToProductData("boolean"); + int type_enumerated = EpsReaderUtils.mapToProductData("enumerated"); + int type_u_byte = EpsReaderUtils.mapToProductData("u-byte"); + int type_integer2 = EpsReaderUtils.mapToProductData("integer2"); + int type_u_integer2 = EpsReaderUtils.mapToProductData("u-integer2"); + int type_integer4 = EpsReaderUtils.mapToProductData("integer4"); + int type_u_integer4 = EpsReaderUtils.mapToProductData("u-integer4"); + int type_integer8 = EpsReaderUtils.mapToProductData("integer8"); + int type_u_integer8 = EpsReaderUtils.mapToProductData("u-integer8"); + + assertEquals(ProductData.TYPE_INT8, type_byte); + assertEquals(ProductData.TYPE_INT8, type_boolean); + assertEquals(ProductData.TYPE_INT8, type_enumerated); + assertEquals(ProductData.TYPE_UINT8, type_u_byte); + assertEquals(ProductData.TYPE_INT16, type_integer2); + assertEquals(ProductData.TYPE_UINT16, type_u_integer2); + assertEquals(ProductData.TYPE_INT32, type_integer4); + assertEquals(ProductData.TYPE_UINT32, type_u_integer4); + assertEquals(ProductData.TYPE_INT64, type_integer8); + assertEquals(ProductData.TYPE_UINT64, type_u_integer8); + } + + @Test + public void testMapToProductDataType_invalid() { + try { + EpsReaderUtils.mapToProductData("invalid"); + fail("Exception expected"); + } catch (IllegalArgumentException expected) { + assertEquals("Unknown data type: invalid", expected.getMessage()); + } + } + + @Test + public void test_scale_appliesFactor() { + int[] values = {10, -20, 30}; + Array input = Array.factory(DataType.INT, new int[]{values.length}, values); + double scaleFactor = 0.1; + + Array result = EpsReaderUtils.scale(input, scaleFactor); + + assertEquals(1.0, result.getDouble(0), 1e-6); + assertEquals(-2.0, result.getDouble(1), 1e-6); + assertEquals(3.0, result.getDouble(2), 1e-6); + } + + @Test + public void test_scale_factorIsOne_returnsSameArray() { + int[] values = {5, 10, 15}; + Array input = Array.factory(DataType.INT, new int[]{values.length}, values); + double scaleFactor = 1.0; + + Array result = EpsReaderUtils.scale(input, scaleFactor); + + assertSame(input, result); + } + + @Test + public void testInitializeArray_int8() { + int numScanLines = 4; + int numFOVs = 20; + Array array = EpsReaderUtils.initializeArray(ProductData.TYPE_INT8, numScanLines, numFOVs); + assertTrue(array instanceof ArrayByte.D2); + assertArrayEquals(new int[]{numScanLines, numFOVs}, array.getShape()); + } + + @Test + public void testInitializeArray_uint8() { + int numScanLines = 4; + int numFOVs = 20; + Array array = EpsReaderUtils.initializeArray(ProductData.TYPE_UINT8, numScanLines, numFOVs); + assertTrue(array instanceof ArrayByte.D2); + assertArrayEquals(new int[]{numScanLines, numFOVs}, array.getShape()); + } + + @Test + public void testInitializeArray_int16() { + int numScanLines = 4; + int numFOVs = 20; + Array array = EpsReaderUtils.initializeArray(ProductData.TYPE_INT16, numScanLines, numFOVs); + assertTrue(array instanceof ArrayShort.D2); + assertArrayEquals(new int[]{numScanLines, numFOVs}, array.getShape()); + } + + @Test + public void testInitializeArray_uint16() { + int numScanLines = 4; + int numFOVs = 20; + Array array = EpsReaderUtils.initializeArray(ProductData.TYPE_UINT16, numScanLines, numFOVs); + assertTrue(array instanceof ArrayShort.D2); + assertArrayEquals(new int[]{numScanLines, numFOVs}, array.getShape()); + } + + @Test + public void testInitializeArray_int32() { + int numScanLines = 4; + int numFOVs = 20; + Array array = EpsReaderUtils.initializeArray(ProductData.TYPE_INT32, numScanLines, numFOVs); + assertTrue(array instanceof ArrayInt.D2); + assertArrayEquals(new int[]{numScanLines, numFOVs}, array.getShape()); + } + + @Test + public void testInitializeArray_uint32() { + int numScanLines = 4; + int numFOVs = 20; + Array array = EpsReaderUtils.initializeArray(ProductData.TYPE_UINT32, numScanLines, numFOVs); + assertTrue(array instanceof ArrayInt.D2); + assertArrayEquals(new int[]{numScanLines, numFOVs}, array.getShape()); + } + + @Test + public void testInitializeArray_int64() { + int numScanLines = 4; + int numFOVs = 20; + Array array = EpsReaderUtils.initializeArray(ProductData.TYPE_INT64, numScanLines, numFOVs); + assertTrue(array instanceof ArrayLong.D2); + assertArrayEquals(new int[]{numScanLines, numFOVs}, array.getShape()); + } + + @Test + public void testInitializeArray_uint64() { + int numScanLines = 4; + int numFOVs = 20; + Array array = EpsReaderUtils.initializeArray(ProductData.TYPE_UINT64, numScanLines, numFOVs); + assertTrue(array instanceof ArrayLong.D2); + assertArrayEquals(new int[]{numScanLines, numFOVs}, array.getShape()); + } + + @Test + public void testInitializeArray_default() { + int numScanLines = 4; + int numFOVs = 20; + Array array = EpsReaderUtils.initializeArray(9999, numScanLines, numFOVs); + assertTrue(array instanceof ArrayDouble.D2); + assertArrayEquals(new int[]{numScanLines, numFOVs}, array.getShape()); + } + + @Test + public void testGetFillValue() { + assertEquals(Byte.MIN_VALUE, EpsReaderUtils.getFillValue("byte")); + assertEquals(255, EpsReaderUtils.getFillValue("u-byte")); + assertEquals(Short.MIN_VALUE, EpsReaderUtils.getFillValue("integer2")); + assertEquals(65535, EpsReaderUtils.getFillValue("u-integer2")); + assertEquals(Integer.MIN_VALUE, EpsReaderUtils.getFillValue("integer4")); + assertEquals(4294967295L, EpsReaderUtils.getFillValue("u-integer4")); + assertEquals(Long.MIN_VALUE, EpsReaderUtils.getFillValue("integer8")); + assertEquals(Long.MAX_VALUE, EpsReaderUtils.getFillValue("u-integer8")); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsVariableCache_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsVariableCache_IO_Test.java new file mode 100644 index 000000000..684b9fa8d --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/EpsVariableCache_IO_Test.java @@ -0,0 +1,107 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import com.bc.fiduceo.IOTestRunner; +import com.bc.fiduceo.reader.amsu_mhs.AMSUA_L1B_Reader; +import com.bc.fiduceo.reader.amsu_mhs.nat.record_types.MDR; +import com.bc.fiduceo.reader.amsu_mhs.nat.record_types.MPHR; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.Array; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +import static org.junit.Assert.*; + +@RunWith(IOTestRunner.class) +public class EpsVariableCache_IO_Test { + + EpsVariableCache cache; + + @Before + public void setUp() throws Exception { + final Path path = Paths.get(getClass().getResource("/com/bc/fiduceo/reader/testFile_amsua_l1b.nat").toURI()); + final byte[] rawDataBuffer = Files.readAllBytes(path); + final VariableRegistry registry = VariableRegistry.load(AMSUA_L1B_Reader.RESOURCE_KEY); + + cache = new EpsVariableCache(rawDataBuffer, registry, EPS_Constants.AMSUA_FOV_COUNT); + } + + @After + public void tearDown() throws Exception { + cache.clear(); + cache = null; + } + + @Test + public void test_getMphr() { + final MPHR mphr = cache.getMPHR(); + + assertNotNull(mphr); + assertEquals(RECORD_CLASS.MPHR, mphr.getHeader().getRecordClass()); + assertEquals(3307, mphr.getPayload().length); + } + + @Test + public void test_getMdrs() { + final List mdrs = cache.getMdrs(); + + assertNotNull(mdrs); + assertFalse(mdrs.isEmpty()); + assertEquals(765, mdrs.size()); + } + + @Test + public void test_getRaw_longitude() { + Array longitude = cache.getRaw("longitude"); + + assertNotNull(longitude); + assertEquals(2, longitude.getRank()); + assertEquals(cache.getMdrs().size(), longitude.getShape()[0]); + assertEquals(EPS_Constants.AMSUA_FOV_COUNT, longitude.getShape()[1]); + + // @todo 2 tb/tb check which data type is it originally + assertEquals(-1680057, longitude.getInt(0)); + assertEquals(1771737, longitude.getInt(10)); + assertEquals(1646190, longitude.getInt(20)); + } + + @Test + public void test_getSceneRadiance_02() { + Array radiance = cache.getRaw("SCENE_RADIANCE_02"); + + assertNotNull(radiance); + assertEquals(2, radiance.getRank()); + + final int height = radiance.getShape()[0]; + final int width = radiance.getShape()[1]; + assertEquals(cache.getMdrs().size(), height); + assertEquals(EPS_Constants.AMSUA_FOV_COUNT, width); + + // @todo 2 tb/tb check which data type is it originally + assertEquals(21796, radiance.getInt(1)); + assertEquals(22334, radiance.getInt(11)); + assertEquals(22465, radiance.getInt(21)); + } + + @Test + public void test_getScaled_latitude() { + Array latitude = cache.getScaled("latitude"); + + assertNotNull(latitude); + assertEquals(2, latitude.getRank()); + + final int height = latitude.getShape()[0]; + final int width = latitude.getShape()[1]; + assertEquals(cache.getMdrs().size(), height); + assertEquals(EPS_Constants.AMSUA_FOV_COUNT, width); + + assertEquals(65.5792, latitude.getDouble(0), 1e-8); + assertEquals(70.2679, latitude.getDouble(10), 1e-8); + assertEquals(72.2871, latitude.getDouble(20), 1e-8); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/GENERIC_RECORD_HEADER_Test.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/GENERIC_RECORD_HEADER_Test.java new file mode 100644 index 000000000..6817aada8 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/GENERIC_RECORD_HEADER_Test.java @@ -0,0 +1,22 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class GENERIC_RECORD_HEADER_Test { + + @Test + public void testParse() { + final byte[] data = {1, 9, 12, 13, 0, 0, 0, 100}; + final GENERIC_RECORD_HEADER genericHeader = GENERIC_RECORD_HEADER.parse(data); + + assertNotNull(genericHeader); + assertEquals(RECORD_CLASS.MPHR, genericHeader.getRecordClass()); + assertEquals((byte) 12, genericHeader.getRecordSubClass()); + assertEquals((byte) 13, genericHeader.getRecordSubClassVersion()); + assertEquals(INSTRUMENT_GROUP.MHS, genericHeader.getInstrumentGroup()); + assertEquals(100, genericHeader.getRecordSize()); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/INSTRUMENT_GROUP_Test.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/INSTRUMENT_GROUP_Test.java new file mode 100644 index 000000000..e30a0d3fe --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/INSTRUMENT_GROUP_Test.java @@ -0,0 +1,52 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import org.junit.Test; + +import static com.bc.fiduceo.reader.amsu_mhs.nat.INSTRUMENT_GROUP.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class INSTRUMENT_GROUP_Test { + + @Test + public void testFromByte() { + assertEquals(GENERIC, INSTRUMENT_GROUP.fromByte((byte) 0)); + assertEquals(AMSUA, INSTRUMENT_GROUP.fromByte((byte) 1)); + assertEquals(ASCAT, INSTRUMENT_GROUP.fromByte((byte) 2)); + assertEquals(ATOVS, INSTRUMENT_GROUP.fromByte((byte) 3)); + assertEquals(AVHRR3, INSTRUMENT_GROUP.fromByte((byte) 4)); + assertEquals(GOME, INSTRUMENT_GROUP.fromByte((byte) 5)); + assertEquals(GRAS, INSTRUMENT_GROUP.fromByte((byte) 6)); + assertEquals(HIRS4, INSTRUMENT_GROUP.fromByte((byte) 7)); + assertEquals(IASI, INSTRUMENT_GROUP.fromByte((byte) 8)); + assertEquals(MHS, INSTRUMENT_GROUP.fromByte((byte) 9)); + assertEquals(SEM, INSTRUMENT_GROUP.fromByte((byte) 10)); + assertEquals(ADCS, INSTRUMENT_GROUP.fromByte((byte) 11)); + assertEquals(SBUV, INSTRUMENT_GROUP.fromByte((byte) 12)); + assertEquals(DUMMY, INSTRUMENT_GROUP.fromByte((byte) 13)); + assertEquals(IASI_L2, INSTRUMENT_GROUP.fromByte((byte) 15)); + assertEquals(ARCHIVE, INSTRUMENT_GROUP.fromByte((byte) 99)); + } + @SuppressWarnings("ResultOfMethodCallIgnored") + @Test + public void testFromByte_invalid() { + try { + INSTRUMENT_GROUP.fromByte((byte) -1); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException expected){ + } + + try { + INSTRUMENT_GROUP.fromByte((byte) 14); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException expected){ + } + + try { + INSTRUMENT_GROUP.fromByte((byte) 68); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException expected){ + } + } + +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/RECORD_CLASS_Test.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/RECORD_CLASS_Test.java new file mode 100644 index 000000000..b370d9c13 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/RECORD_CLASS_Test.java @@ -0,0 +1,40 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import org.junit.Test; + +import static com.bc.fiduceo.reader.amsu_mhs.nat.RECORD_CLASS.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class RECORD_CLASS_Test { + + @Test + public void testFromByte() { + assertEquals(RESERVED, RECORD_CLASS.fromByte((byte) 0)); + assertEquals(MPHR, RECORD_CLASS.fromByte((byte) 1)); + assertEquals(SPHR, RECORD_CLASS.fromByte((byte) 2)); + assertEquals(IPR, RECORD_CLASS.fromByte((byte) 3)); + assertEquals(GEADR, RECORD_CLASS.fromByte((byte) 4)); + assertEquals(GIADR, RECORD_CLASS.fromByte((byte) 5)); + assertEquals(VEADR, RECORD_CLASS.fromByte((byte) 6)); + assertEquals(VIADR, RECORD_CLASS.fromByte((byte) 7)); + assertEquals(MDR, RECORD_CLASS.fromByte((byte) 8)); + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @Test + public void testFromByte_invalid() { + try { + RECORD_CLASS.fromByte((byte) -1); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException expected){ + } + + try { + RECORD_CLASS.fromByte((byte) 9); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException expected) { + } + } + +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/RecordFactoryTest.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/RecordFactoryTest.java new file mode 100644 index 000000000..fce05c37c --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/RecordFactoryTest.java @@ -0,0 +1,91 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import com.bc.fiduceo.reader.amsu_mhs.nat.record_types.*; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class RecordFactoryTest { + + @Test + public void parseRecords() { + byte[] recordMphr = createDummyRecord(RECORD_CLASS.MPHR, 24, (byte) 0x11); + byte[] recordReserved = createDummyRecord(RECORD_CLASS.RESERVED, 28, (byte) 0x22); + byte[] recordSPHR = createDummyRecord(RECORD_CLASS.SPHR, 28, (byte) 0x55); + byte[] recordMdr1 = createDummyRecord(RECORD_CLASS.MDR, 64, (byte) 0x33); + byte[] recordMdr2 = createDummyRecord(RECORD_CLASS.MDR, 128, (byte) 0x44); + + byte[] allBytes = new byte[recordMphr.length + recordReserved.length + recordSPHR.length + recordMdr1.length + recordMdr2.length]; + System.arraycopy(recordMphr, 0, allBytes, 0, recordMphr.length); + System.arraycopy(recordReserved, 0, allBytes, recordMphr.length, recordReserved.length); + System.arraycopy(recordSPHR, 0, allBytes, recordMphr.length + recordReserved.length, recordSPHR.length); + System.arraycopy(recordMdr1, 0, allBytes, recordMphr.length + recordReserved.length + recordSPHR.length, recordMdr1.length); + System.arraycopy(recordMdr2, 0, allBytes, recordMphr.length + recordReserved.length + recordSPHR.length + recordMdr1.length, recordMdr2.length); + + List records = RecordFactory.parseRecords(allBytes); + + assertEquals(5, records.size()); + assertTrue(records.get(0) instanceof MPHR); + assertTrue(records.get(1).getClass() == Record.class); + assertTrue(records.get(2) instanceof SPHR); + assertTrue(records.get(3) instanceof MDR); + assertTrue(records.get(4) instanceof MDR); + } + + @Test + public void createRecord() { + GENERIC_RECORD_HEADER headerMPHR = new GENERIC_RECORD_HEADER(RECORD_CLASS.MPHR, INSTRUMENT_GROUP.MHS, (byte) 3, (byte) 6, 42); + GENERIC_RECORD_HEADER headerSPHR = new GENERIC_RECORD_HEADER(RECORD_CLASS.SPHR, INSTRUMENT_GROUP.MHS, (byte) 3, (byte) 6, 42); + GENERIC_RECORD_HEADER headerIPR = new GENERIC_RECORD_HEADER(RECORD_CLASS.IPR, INSTRUMENT_GROUP.MHS, (byte) 3, (byte) 6, 42); + GENERIC_RECORD_HEADER headerGEADR = new GENERIC_RECORD_HEADER(RECORD_CLASS.GEADR, INSTRUMENT_GROUP.MHS, (byte) 3, (byte) 6, 42); + GENERIC_RECORD_HEADER headerGIADR = new GENERIC_RECORD_HEADER(RECORD_CLASS.GIADR, INSTRUMENT_GROUP.MHS, (byte) 3, (byte) 6, 42); + GENERIC_RECORD_HEADER headerVEADR = new GENERIC_RECORD_HEADER(RECORD_CLASS.VEADR, INSTRUMENT_GROUP.MHS, (byte) 3, (byte) 6, 42); + GENERIC_RECORD_HEADER headerVIADR = new GENERIC_RECORD_HEADER(RECORD_CLASS.VIADR, INSTRUMENT_GROUP.MHS, (byte) 3, (byte) 6, 42); + GENERIC_RECORD_HEADER headerMDR = new GENERIC_RECORD_HEADER(RECORD_CLASS.MDR, INSTRUMENT_GROUP.MHS, (byte) 4, (byte) 7, 42); + GENERIC_RECORD_HEADER headerRecord = new GENERIC_RECORD_HEADER(RECORD_CLASS.RESERVED, INSTRUMENT_GROUP.MHS, (byte) 5, (byte) 8, 42); + byte[] payload = new byte[22]; + + Record recordMPHR = RecordFactory.createRecord(headerMPHR, payload); + Record recordSPHR = RecordFactory.createRecord(headerSPHR, payload); + Record recordIPR = RecordFactory.createRecord(headerIPR, payload); + Record recordGEADR = RecordFactory.createRecord(headerGEADR, payload); + Record recordGIADR = RecordFactory.createRecord(headerGIADR, payload); + Record recordVEADR = RecordFactory.createRecord(headerVEADR, payload); + Record recordVIADR = RecordFactory.createRecord(headerVIADR, payload); + Record recordMDR = RecordFactory.createRecord(headerMDR, payload); + Record recordDefault = RecordFactory.createRecord(headerRecord, payload); + + assertTrue(recordMPHR instanceof MPHR); + assertTrue(recordSPHR instanceof SPHR); + assertTrue(recordIPR instanceof IPR); + assertTrue(recordGEADR instanceof GEADR); + assertTrue(recordGIADR instanceof GIADR); + assertTrue(recordVEADR instanceof VEADR); + assertTrue(recordVIADR instanceof VIADR); + assertTrue(recordMDR instanceof MDR); + assertTrue(recordDefault.getClass() == Record.class); + + assertEquals(payload.length, recordMPHR.getPayload().length); + } + + private byte[] createDummyRecord(RECORD_CLASS recordClass, int size, byte fillValue) { + byte[] record = new byte[size]; + ByteBuffer buffer = ByteBuffer.wrap(record); + + buffer.put((byte) recordClass.ordinal()); // RECORD_CLASS + buffer.put((byte) 0); // INSTRUMENT_GROUP dummy + buffer.put((byte) 0); // RECORD_SUBCLASS + buffer.put((byte) 1); // RECORD_SUBCLASS_VERSION + buffer.putInt(size); // RECORD_SIZE + + for (int i = 20; i < size; i++) { + record[i] = fillValue; + } + return record; + } + +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableDefinitionTest.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableDefinitionTest.java new file mode 100644 index 000000000..9f2393ef4 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableDefinitionTest.java @@ -0,0 +1,16 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class VariableDefinitionTest { + + @Test + public void testConstruction() { + final VariableDefinition definition = new VariableDefinition(); + assertEquals(1.0, definition.getScale_factor(), 1e-8); + assertEquals("", definition.getUnits()); + assertEquals("ARRAY", definition.getData_layout()); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableRegistryTest.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableRegistryTest.java new file mode 100644 index 000000000..f0d0e71fd --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/VariableRegistryTest.java @@ -0,0 +1,211 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat; + +import com.bc.fiduceo.reader.amsu_mhs.AMSUA_L1B_Reader; +import com.bc.fiduceo.reader.amsu_mhs.MHS_L1B_Reader; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.*; + + +public class VariableRegistryTest { + + @Test + public void test_load_success() { + VariableRegistry mhsRegistry = VariableRegistry.load(MHS_L1B_Reader.RESOURCE_KEY); + VariableRegistry amsuaRegistry = VariableRegistry.load(AMSUA_L1B_Reader.RESOURCE_KEY); + + assertNotNull(mhsRegistry); + assertNotNull(amsuaRegistry); + + Map mhsVars = mhsRegistry.getVariables(); + Map amsuaVars = amsuaRegistry.getVariables(); + + assertFalse(mhsVars.isEmpty()); + assertFalse(amsuaVars.isEmpty()); + } + + @Test + public void test_load_error() { + try { + VariableRegistry.load("xyz"); + fail("Expected exception"); + } catch (RuntimeException expected) { + assertEquals("Resource not found: xyz/variables.json", expected.getMessage()); + } + } + + @Test + public void test_load_contents_mhs_l1b() { + VariableRegistry mhsRegistry = VariableRegistry.load(MHS_L1B_Reader.RESOURCE_KEY); + + assertNotNull(mhsRegistry); + + Map mhsVars = mhsRegistry.getVariables(); + + assertEquals(15, mhsVars.size()); + + final VariableDefinition sceneRadiance01 = mhsVars.get("SCENE_RADIANCES_01"); + assertEquals("integer4", sceneRadiance01.getData_type()); + assertEquals(83, sceneRadiance01.getOffset()); + assertEquals(5, sceneRadiance01.getStride()); + assertEquals(.0000001, sceneRadiance01.getScale_factor(), 1e-10); + assertEquals("mW/m2/sr/cm-1", sceneRadiance01.getUnits()); + + final VariableDefinition sceneRadiance05 = mhsVars.get("SCENE_RADIANCES_05"); + assertEquals("integer4", sceneRadiance05.getData_type()); + assertEquals(99, sceneRadiance05.getOffset()); + assertEquals(5, sceneRadiance05.getStride()); + assertEquals(.0000001, sceneRadiance05.getScale_factor(), 1e-10); + assertEquals("mW/m2/sr/cm-1", sceneRadiance05.getUnits()); + + final VariableDefinition dataQuality = mhsVars.get("FOV_DATA_QUALITY"); + assertEquals("integer4", dataQuality.getData_type()); + assertEquals(1883, dataQuality.getOffset()); + assertEquals(1, dataQuality.getStride()); + assertEquals(1, dataQuality.getScale_factor(), 1e-10); + assertEquals("", dataQuality.getUnits()); + + final VariableDefinition timeAttitude = mhsVars.get("TIME_ATTITUDE"); + assertEquals("u-integer4", timeAttitude.getData_type()); + assertEquals(2580, timeAttitude.getOffset()); + assertEquals(1, timeAttitude.getStride()); + assertEquals("s", timeAttitude.getUnits()); + assertEquals("VECTOR", timeAttitude.getData_layout()); + + final VariableDefinition latitude = mhsVars.get("latitude"); + assertEquals("integer4", latitude.getData_type()); + assertEquals(3318, latitude.getOffset()); + assertEquals(2, latitude.getStride()); + assertEquals(.0001, latitude.getScale_factor(), 1e-10); + assertEquals("degree", latitude.getUnits()); + + final VariableDefinition longitude = mhsVars.get("longitude"); + assertEquals("integer4", longitude.getData_type()); + assertEquals(3322, longitude.getOffset()); + assertEquals(2, longitude.getStride()); + assertEquals(.0001, longitude.getScale_factor(), 1e-10); + assertEquals("degree", longitude.getUnits()); + } + + @Test + public void test_load_contents_amsua_l1b() { + VariableRegistry amsuaRegistry = VariableRegistry.load(AMSUA_L1B_Reader.RESOURCE_KEY); + + assertNotNull(amsuaRegistry); + + Map amsuaVars = amsuaRegistry.getVariables(); + + assertEquals(24, amsuaVars.size()); + + final VariableDefinition sceneRadiance01 = amsuaVars.get("SCENE_RADIANCE_01"); + assertEquals("integer4", sceneRadiance01.getData_type()); + assertEquals(22, sceneRadiance01.getOffset()); + assertEquals(15, sceneRadiance01.getStride()); + assertEquals(.0000001, sceneRadiance01.getScale_factor(), 1e-10); + assertEquals("mW/m2/sr/cm-1", sceneRadiance01.getUnits()); + + final VariableDefinition sceneRadiance15 = amsuaVars.get("SCENE_RADIANCE_15"); + assertEquals("integer4", sceneRadiance15.getData_type()); + assertEquals(78, sceneRadiance15.getOffset()); + assertEquals(15, sceneRadiance15.getStride()); + assertEquals(.0000001, sceneRadiance15.getScale_factor(), 1e-10); + assertEquals("mW/m2/sr/cm-1", sceneRadiance15.getUnits()); + + final VariableDefinition solarZenith = amsuaVars.get("solar_zenith_angle"); + assertEquals("integer2", solarZenith.getData_type()); + assertEquals(1842, solarZenith.getOffset()); + assertEquals(4, solarZenith.getStride()); + assertEquals(.01, solarZenith.getScale_factor(), 1e-10); + assertEquals("degree", solarZenith.getUnits()); + + final VariableDefinition satelliteZenith = amsuaVars.get("satellite_zenith_angle"); + assertEquals("integer2", satelliteZenith.getData_type()); + assertEquals(1844, satelliteZenith.getOffset()); + assertEquals(4, satelliteZenith.getStride()); + assertEquals(.01, satelliteZenith.getScale_factor(), 1e-10); + assertEquals("degree", satelliteZenith.getUnits()); + + final VariableDefinition solarAzimuth = amsuaVars.get("solar_azimuth_angle"); + assertEquals("integer2", solarAzimuth.getData_type()); + assertEquals(1846, solarAzimuth.getOffset()); + assertEquals(4, solarAzimuth.getStride()); + assertEquals(.01, solarAzimuth.getScale_factor(), 1e-10); + assertEquals("degree", solarAzimuth.getUnits()); + + final VariableDefinition satelliteAzimuth = amsuaVars.get("satellite_azimuth_angle"); + assertEquals("integer2", satelliteAzimuth.getData_type()); + assertEquals(1848, satelliteAzimuth.getOffset()); + assertEquals(4, satelliteAzimuth.getStride()); + assertEquals(.01, satelliteAzimuth.getScale_factor(), 1e-10); + assertEquals("degree", satelliteAzimuth.getUnits()); + + final VariableDefinition latitude = amsuaVars.get("latitude"); + assertEquals("integer4", latitude.getData_type()); + assertEquals(2082, latitude.getOffset()); + assertEquals(2, latitude.getStride()); + assertEquals(.0001, latitude.getScale_factor(), 1e-10); + assertEquals("degree", latitude.getUnits()); + + final VariableDefinition longitude = amsuaVars.get("longitude"); + assertEquals("integer4", longitude.getData_type()); + assertEquals(2086, longitude.getOffset()); + assertEquals(2, longitude.getStride()); + assertEquals(.0001, longitude.getScale_factor(), 1e-10); + assertEquals("degree", longitude.getUnits()); + + final VariableDefinition surfaceProperties = amsuaVars.get("SURFACE_PROPERTIES"); + assertEquals("integer2", surfaceProperties.getData_type()); + assertEquals(2322, surfaceProperties.getOffset()); + assertEquals(1, surfaceProperties.getStride()); + assertEquals(1.0, surfaceProperties.getScale_factor(), 1e-10); + assertEquals("", surfaceProperties.getUnits()); + assertEquals("0, 1, 2", surfaceProperties.getFlag_values()); + assertEquals("water mixed_coast land", surfaceProperties.getFlag_meanings()); + + final VariableDefinition terrainElevation = amsuaVars.get("TERRAIN_ELEVATION"); + assertEquals("integer2", terrainElevation.getData_type()); + assertEquals(2382, terrainElevation.getOffset()); + assertEquals(1, terrainElevation.getStride()); + assertEquals(1.0, terrainElevation.getScale_factor(), 1e-10); + assertEquals("m", terrainElevation.getUnits()); + + final VariableDefinition timeAttitude = amsuaVars.get("TIME_ATTITUDE"); + assertEquals("u-integer4", timeAttitude.getData_type()); + assertEquals(1824, timeAttitude.getOffset()); + assertEquals(1, timeAttitude.getStride()); + assertEquals("s", timeAttitude.getUnits()); + assertEquals("VECTOR", timeAttitude.getData_layout()); + } + + @Test + public void test_getVariableDef_success() { + VariableRegistry mhsRegistry = VariableRegistry.load(MHS_L1B_Reader.RESOURCE_KEY); + assertNotNull(mhsRegistry); + + VariableDefinition lon = mhsRegistry.getVariableDef("longitude"); + assertNotNull(lon); + assertEquals(3322, lon.getOffset()); + } + + @Test + public void test_getVariableDef_error() { + VariableRegistry mhsRegistry = VariableRegistry.load(MHS_L1B_Reader.RESOURCE_KEY); + assertNotNull(mhsRegistry); + + try { + VariableDefinition lon = mhsRegistry.getVariableDef("not_existing_variable"); + fail("Expected exception"); + } catch (IllegalArgumentException expected) { + assertEquals("Variable not defined: not_existing_variable", expected.getMessage()); + } + + try { + VariableDefinition lon = mhsRegistry.getVariableDef("SCENE_RADIANCE_16"); + fail("Expected exception"); + } catch (IllegalArgumentException expected) { + assertEquals("Variable not defined: SCENE_RADIANCE_16", expected.getMessage()); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/ASMUSA_MDR_Test.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/ASMUSA_MDR_Test.java new file mode 100644 index 000000000..3368f817b --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/ASMUSA_MDR_Test.java @@ -0,0 +1,30 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat.record_types; + +import com.bc.fiduceo.reader.amsu_mhs.nat.GENERIC_RECORD_HEADER; +import org.apache.commons.lang3.Conversion; +import org.junit.Test; + +public class ASMUSA_MDR_Test { + + + + @Test + public void testGetVariable() { + final byte[] payload = new byte[3464]; + payload[0] = (byte) 8; // MDR + payload[1] = (byte) 1; // AMSUA + payload[4] = (byte) 20; // RECORD_SIZE + + int writePointer = 2082; // start of EARTH_LOCATION + for (int i = 0; i < 30; i++) { + int offset = i * 8; // two times four bytes [latitude,longitude] + Conversion.intToByteArray(i, 0, payload, writePointer + offset, 4); + Conversion.intToByteArray(100 + i, 0, payload, writePointer + offset + 4, 4); + } + + final GENERIC_RECORD_HEADER recordHeader = GENERIC_RECORD_HEADER.parse(payload); + + final ASMUSA_MDR asmsuaMdr = new ASMUSA_MDR(recordHeader, payload); + final int[] rawData = asmsuaMdr.parseVariable("longitude"); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/MPHRTest.java b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/MPHRTest.java new file mode 100644 index 000000000..b2bf0456b --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/amsu_mhs/nat/record_types/MPHRTest.java @@ -0,0 +1,76 @@ +package com.bc.fiduceo.reader.amsu_mhs.nat.record_types; + +import com.bc.fiduceo.reader.amsu_mhs.nat.EPS_Constants; +import org.junit.Test; + +import java.io.IOException; +import java.util.Date; + +import static org.junit.Assert.*; + +public class MPHRTest { + + @Test + public void testExtractStartAndEndTime() throws IOException { + final byte[] payLoad = new byte[3307]; + String sensingStart = "SENSING_START = 20250820060350Z"; + String sensingEnd = "SENSING_END = 20250820074550Z"; + + System.arraycopy(sensingStart.getBytes(), 0, payLoad, 700, sensingStart.getBytes().length); + System.arraycopy(sensingEnd.getBytes(), 0, payLoad, 748, sensingStart.getBytes().length); + MPHR mphr = new MPHR(null, payLoad); + + Date sensingStartDate = mphr.getSensingStart(); + Date sensingEndDate = mphr.getSensingStop(); + + assertEquals(1755669830000L, sensingStartDate.getTime()); + assertEquals(1755675950000L, sensingEndDate.getTime()); + assertTrue(sensingStartDate.before(sensingEndDate)); + } + + @Test + public void testParseError() { + final byte[] payLoad = new byte[3307]; + String sensingStart = "SENSING_START = 20250820060350X"; + String sensingEnd = "SENSING_END = 20250820074550X"; + + System.arraycopy(sensingStart.getBytes(), 0, payLoad, 700, sensingStart.getBytes().length); + System.arraycopy(sensingEnd.getBytes(), 0, payLoad, 748, sensingStart.getBytes().length); + MPHR mphr = new MPHR(null, payLoad); + + Exception ex = assertThrows(IOException.class, mphr::getSensingStart); + assertEquals("Could not parse time: 20250820060350X", ex.getMessage()); + } + + @Test + public void testGetDate_invalidKey() { + final byte[] payLoad = new byte[3307]; + MPHR mphr = new MPHR(null, payLoad); + + try { + mphr.getDate("FANCY_DATE"); + fail("IllegalStateException expected"); + } catch (IllegalStateException | IOException expected) { + } + } + + @Test + public void testInvalidPayload() { + final byte[] payLoad = new byte[3307]; + MPHR mphr = new MPHR(null, payLoad); + + Exception ex = assertThrows(IllegalStateException.class, mphr::getSensingStart); + assertEquals("Invalid attribute formatting: ", ex.getMessage()); + } + + @Test + public void testGetProductName() { + final byte[] payLoad = new byte[3307]; + final String productName = "PRODUCT_NAME = The_thing_how_we_call_it_version7"; + byte[] productNameBytes = productName.getBytes(); + System.arraycopy(productNameBytes, 0, payLoad, EPS_Constants.GENERIC_RECORD_HEADER_SIZE, productNameBytes.length); + final MPHR mphr = new MPHR(null, payLoad); + + assertEquals("The_thing_how_we_call_it_version7", mphr.getProductName()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/fiduceo_fcdr/HIRS_FCDR_Reader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/fiduceo_fcdr/HIRS_FCDR_Reader_IO_Test.java index 54ccfb46c..798b35841 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/fiduceo_fcdr/HIRS_FCDR_Reader_IO_Test.java +++ b/core/src/test/java/com/bc/fiduceo/reader/fiduceo_fcdr/HIRS_FCDR_Reader_IO_Test.java @@ -515,7 +515,7 @@ public void testGetPixelLocator() throws IOException { assertNotNull(pixelLocation); assertEquals(1, pixelLocation.length); - assertEquals(38.317269513397704, pixelLocation[0].getX(), 1e-8); + assertEquals(38.317269513397704, pixelLocation[0].getX(), 1e-6); assertEquals(781.666865947429, pixelLocation[0].getY(), 1e-8); } finally { reader.close(); @@ -540,7 +540,7 @@ public void testGetPixelLocator_N09() throws IOException { assertNotNull(pixelLocation); assertEquals(1, pixelLocation.length); - assertEquals(38.50497313441438, pixelLocation[0].getX(), 1e-8); + assertEquals(38.50497313441438, pixelLocation[0].getX(), 1e-6); assertEquals(783.6241097278424, pixelLocation[0].getY(), 1e-8); } finally { reader.close(); @@ -567,7 +567,7 @@ public void testGetSubScenePixelLocator() throws IOException { assertNotNull(pixelLocation); assertEquals(1, pixelLocation.length); - assertEquals(33.59545482099081, pixelLocation[0].getX(), 1e-8); + assertEquals(33.59545482099081, pixelLocation[0].getX(), 1e-6); assertEquals(416.5860033509568, pixelLocation[0].getY(), 1e-8); } finally { diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/CsvFormatConfigTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/CsvFormatConfigTest.java new file mode 100644 index 000000000..5dfc43f6f --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/CsvFormatConfigTest.java @@ -0,0 +1,278 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import org.esa.snap.core.datamodel.ProductData; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; + +public class CsvFormatConfigTest { + + @Test + public void testLoadCsvFormatConfig_SM_success() { + CsvFormatConfig config = CsvFormatConfig.loadConfig(GenericCsvHelper.RESOURCE_KEY_NDBC_SM); + + assertEquals("NDBC_SM", config.getName()); + assertEquals("space", config.getDelimiter()); + assertEquals('#', config.getCommentChar()); + assertEquals("\\w{5}h\\d{4}.txt", config.getRegex()); + assertEquals("longitude", config.getLongitudeName()); + assertEquals("latitude", config.getLatitudeName()); + assertEquals("time", config.getTimeName()); + assertEquals(5, config.getTimeVars().size()); + assertTrue(config.isLocationFromStationDatabase()); + + List vars = config.getVariables(); + assertEquals(18, vars.size()); + + GenericVariable var5 = vars.get(5); + assertEquals("WDIR", var5.getName()); + assertEquals("short", var5.getType()); + assertEquals(ProductData.TYPE_INT16, var5.getProductData()); + assertEquals('v', var5.getOrigin()); + assertEquals(999, var5.getFillValue(), .00001); + assertEquals("degT", var5.getUnits()); + assertEquals("Ten-minute average wind direction measurements in degrees clockwise from true North.", var5.getLongName()); + assertEquals("wind_from_direction", var5.getCfStandard()); + + StationDatabase database = config.getStationDatabase(); + assertNotNull(database); + + String stationIdIdentifier = database.getPrimaryId(); + assertNotNull(stationIdIdentifier); + assertEquals("id", stationIdIdentifier); + + List stationVars = database.getVariables(); + assertEquals(9, stationVars.size()); + + assertEquals("id", stationVars.get(0).getName()); + assertEquals("latitude", stationVars.get(1).getName()); + assertEquals("longitude", stationVars.get(2).getName()); + assertEquals("station_type", stationVars.get(3).getName()); + assertEquals("measurement_type", stationVars.get(4).getName()); + assertEquals("anemometer_height", stationVars.get(5).getName()); + assertEquals("air_temp_height", stationVars.get(6).getName()); + assertEquals("barometer_height", stationVars.get(7).getName()); + assertEquals("sst_depth", stationVars.get(8).getName()); + + GenericVariable stationAirTempHeight = stationVars.get(6); + assertEquals("float", stationAirTempHeight.getType()); + assertEquals(ProductData.TYPE_FLOAT32, stationAirTempHeight.getProductData()); + assertEquals('s', stationAirTempHeight.getOrigin()); + assertNull(stationAirTempHeight.getFillValue()); + assertEquals("m", stationAirTempHeight.getUnits()); + assertEquals("Height of instrument above site elevation", stationAirTempHeight.getLongName()); + assertNull(stationAirTempHeight.getCfStandard()); + + List> stations = database.getStations(); + assertNotNull(stations); + assertFalse(stations.isEmpty()); + assertEquals(444, stations.size()); + + List station = stations.get(443); + assertEquals(9, station.size()); + + assertEquals("YGNN6", station.get(0)); + assertEquals(43.262, station.get(1)); + assertEquals(-79.064, station.get(2)); + assertEquals(5, station.get(3)); + assertEquals(1, station.get(4)); + assertEquals(10.00, station.get(5)); + assertEquals(6.00, station.get(6)); + assertEquals(75.30, station.get(7)); + assertNull(station.get(8)); + } + + @Test + public void testLoadCsvFormatConfig_CW_success() { + CsvFormatConfig config = CsvFormatConfig.loadConfig(GenericCsvHelper.RESOURCE_KEY_NDBC_CW); + + assertEquals("NDBC_CW", config.getName()); + assertEquals("space", config.getDelimiter()); + assertEquals('#', config.getCommentChar()); + assertEquals("\\w{5}c\\d{4}.txt", config.getRegex()); + assertEquals("longitude", config.getLongitudeName()); + assertEquals("latitude", config.getLatitudeName()); + assertEquals("time", config.getTimeName()); + assertEquals(5, config.getTimeVars().size()); + assertTrue(config.isLocationFromStationDatabase()); + + List vars = config.getVariables(); + assertEquals(10, vars.size()); + + GenericVariable var9 = vars.get(9); + assertEquals("GTIME", var9.getName()); + assertEquals("short", var9.getType()); + assertEquals(ProductData.TYPE_INT16, var9.getProductData()); + assertEquals('v', var9.getOrigin()); + assertEquals(9999, var9.getFillValue(), .00001); + assertEquals("hhmm", var9.getUnits()); + assertEquals("The minute of the hour that the GSP occurred, reported at the last hourly 10-minute segment.", var9.getLongName()); + assertNull(var9.getCfStandard()); + + StationDatabase database = config.getStationDatabase(); + assertNotNull(database); + + String stationIdIdentifier = database.getPrimaryId(); + assertNotNull(stationIdIdentifier); + assertEquals("id", stationIdIdentifier); + + List stationVars = database.getVariables(); + assertEquals(6, stationVars.size()); + + assertEquals("id", stationVars.get(0).getName()); + assertEquals("latitude", stationVars.get(1).getName()); + assertEquals("longitude", stationVars.get(2).getName()); + assertEquals("station_type", stationVars.get(3).getName()); + assertEquals("measurement_type", stationVars.get(4).getName()); + assertEquals("anemometer_height", stationVars.get(5).getName()); + + GenericVariable stationAnemometerHeight = stationVars.get(5); + assertEquals("float", stationAnemometerHeight.getType()); + assertEquals(ProductData.TYPE_FLOAT32, stationAnemometerHeight.getProductData()); + assertEquals('s', stationAnemometerHeight.getOrigin()); + assertNull(stationAnemometerHeight.getFillValue()); + assertEquals("m", stationAnemometerHeight.getUnits()); + assertEquals("Height of instrument above site elevation", stationAnemometerHeight.getLongName()); + assertNull(stationAnemometerHeight.getCfStandard()); + + List> stations = database.getStations(); + assertNotNull(stations); + assertFalse(stations.isEmpty()); + assertEquals(115, stations.size()); + + List station = stations.get(114); + assertEquals(6, station.size()); + + assertEquals("STDM4", station.get(0)); + assertEquals(47.184, station.get(1)); + assertEquals(-87.225, station.get(2)); + assertEquals(5, station.get(3)); + assertEquals(0, station.get(4)); + assertEquals(35.2, station.get(5)); + } + + @Test + public void testLoadCsvFormatConfig_GBOV_success() { + CsvFormatConfig config = CsvFormatConfig.loadConfig(GenericCsvHelper.RESOURCE_KEY_GBOV); + + assertEquals("GBOV", config.getName()); + assertEquals(";", config.getDelimiter()); + assertEquals('#', config.getCommentChar()); + assertEquals("^GBOV__(.*?)__(.*?)__([0-9]{8}T[0-9]{6}Z)__([0-9]{8}T[0-9]{6}Z)\\.csv$", config.getRegex()); + assertEquals("Lon_IS", config.getLongitudeName()); + assertEquals("Lat_IS", config.getLatitudeName()); + assertEquals("TIME_IS", config.getTimeName()); + assertNull(config.getTimeVars()); + assertTrue(config.isLocationFromStationDatabase()); + + List vars = config.getVariables(); + assertEquals(61, vars.size()); + + GenericVariable var9 = vars.get(7); + assertEquals("RM6_down_flag", var9.getName()); + assertEquals("short", var9.getType()); + assertEquals(ProductData.TYPE_INT16, var9.getProductData()); + assertEquals('v', var9.getOrigin()); + assertEquals((short) -999, var9.getFillValue(), .00001); + assertEquals("1", var9.getUnits()); + assertEquals("down_flag", var9.getAncillaryVariables()); + + StationDatabase database = config.getStationDatabase(); + assertNotNull(database); + + String siteIdIdentifier = database.getPrimaryId(); + assertNotNull(siteIdIdentifier); + assertEquals("site", siteIdIdentifier); + String stationIdIdentifier = database.getSecondaryId(); + assertNotNull(stationIdIdentifier); + assertEquals("station", stationIdIdentifier); + + List stationVars = database.getVariables(); + assertEquals(6, stationVars.size()); + + assertEquals("site", stationVars.get(0).getName()); + assertEquals("station", stationVars.get(1).getName()); + assertEquals("elevation", stationVars.get(2).getName()); + assertEquals("IGBP_class", stationVars.get(3).getName()); + assertEquals("Lat_IS", stationVars.get(4).getName()); + assertEquals("Lon_IS", stationVars.get(5).getName()); + + GenericVariable stationLatitude = stationVars.get(4); + assertEquals("float", stationLatitude.getType()); + assertEquals(ProductData.TYPE_FLOAT32, stationLatitude.getProductData()); + assertEquals('s', stationLatitude.getOrigin()); + assertNull(stationLatitude.getFillValue()); + assertEquals("degree_north", stationLatitude.getUnits()); + assertEquals("Latitude is positive northward; its units of degree_north (or equivalent) indicate this explicitly. In a latitude-longitude system defined with respect to a rotated North Pole, the standard name of grid_latitude should be used instead of latitude. Grid latitude is positive in the grid-northward direction, but its units should be plain degree.", stationLatitude.getLongName()); + assertEquals("latitude", stationLatitude.getCfStandard()); + + List> stations = database.getStations(); + assertNotNull(stations); + assertFalse(stations.isEmpty()); + + assertEquals(781, stations.size()); + + List station = stations.get(780); + assertEquals(6, station.size()); + + assertEquals("Payerne", station.get(0)); + assertEquals("Payerne", station.get(1)); + assertEquals(637, station.get(2)); + assertEquals("Evergreen Needleleaf", station.get(3)); + assertEquals(46.815, station.get(4)); + assertEquals(6.944, station.get(5)); + + } + + @Test + public void testLoadCsvFormatConfig_resourceNotFound() { + try { + CsvFormatConfig.loadConfig("YY"); + fail(); + } catch (Exception e) { + assertEquals("Resource not found: YY_config.json", e.getMessage()); + } + } + + @Test + public void testLoadCsvFormatConfig_failedToLoadConfig() { + try { + CsvFormatConfig.loadConfig("XX"); + fail(); + } catch (Exception e) { + assertTrue(e.getMessage().contains("Failed to load config from: ")); + } + } + + @Test + public void testGetAllVariables_GBOV() { + CsvFormatConfig config = CsvFormatConfig.loadConfig(GenericCsvHelper.RESOURCE_KEY_GBOV); + + List vars = config.getAllVariables(); + assertNotNull(vars); + assertFalse(vars.isEmpty()); + assertEquals(67, vars.size()); + } + + @Test + public void testGetAllVariables_NDBC_CW() { + CsvFormatConfig config = CsvFormatConfig.loadConfig(GenericCsvHelper.RESOURCE_KEY_NDBC_CW); + + List vars = config.getAllVariables(); + assertNotNull(vars); + assertFalse(vars.isEmpty()); + assertEquals(12, vars.size()); + } + + @Test + public void testGetAllVariables_NDBC_SM() { + CsvFormatConfig config = CsvFormatConfig.loadConfig(GenericCsvHelper.RESOURCE_KEY_NDBC_SM); + + List vars = config.getAllVariables(); + assertNotNull(vars); + assertFalse(vars.isEmpty()); + assertEquals(23, vars.size()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GbovReaderPluginTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GbovReaderPluginTest.java new file mode 100644 index 000000000..a910ab7ff --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GbovReaderPluginTest.java @@ -0,0 +1,39 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + + +public class GbovReaderPluginTest { + + private GbovReaderPlugin plugin; + + @Before + public void setUp() { + plugin = new GbovReaderPlugin(); + } + + @Test + public void testCreateReader() { + final Reader reader = plugin.createReader(null); + assertNotNull(reader); + assertTrue(reader instanceof GenericCsvReader); + } + + @Test + public void testGetSupportedSensorKeys() { + final String[] expected = {"gbov"}; + + final String[] sensorKeys = plugin.getSupportedSensorKeys(); + assertArrayEquals(expected, sensorKeys); + } + + @Test + public void testGetDataType() { + assertEquals(DataType.INSITU, plugin.getDataType()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvHelperTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvHelperTest.java new file mode 100644 index 000000000..b33cca56c --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvHelperTest.java @@ -0,0 +1,161 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import org.junit.Test; +import ucar.ma2.DataType; + +import java.io.File; + +import static org.junit.Assert.*; + + +public class GenericCsvHelperTest { + + @Test + public void testExtractYearMonthDayFromFilename_NDBC_success() { + int[] ymd = GenericCsvHelper.extractYearMonthDayFromFilename("12345h2026.txt", GenericCsvHelper.RESOURCE_KEY_NDBC_CW); + assertEquals(3, ymd.length); + assertEquals(2026, ymd[0]); + assertEquals(1, ymd[1]); + assertEquals(1, ymd[2]); + } + + @Test + public void testExtractYearMonthDayFromFilename_GBOV_success() { + int[] ymd = GenericCsvHelper.extractYearMonthDayFromFilename("GBOV__site--name__station--name__20021023T000000Z__20021031T235900Z.csv", GenericCsvHelper.RESOURCE_KEY_GBOV); + assertEquals(3, ymd.length); + assertEquals(2002, ymd[0]); + assertEquals(10, ymd[1]); + assertEquals(23, ymd[2]); + } + + @Test + public void testExtractYearMonthDayFromFilename_error() { + try { + GenericCsvHelper.extractYearMonthDayFromFilename("12345h2026.txt", "not_supported_format_key"); + fail(); + } catch (IllegalStateException e) { + assertEquals("Unsupported format for file: '12345h2026.txt' and resourceKey 'not_supported_format_key'.", e.getMessage()); + } + } + + @Test + public void testGetPrimaryIdFromFilename_NDBC_success() { + File file = new File("54321c2017.txt"); + String resKey1 = GenericCsvHelper.RESOURCE_KEY_NDBC_SM; + String resKey2 = GenericCsvHelper.RESOURCE_KEY_NDBC_CW; + + String stationId1 = GenericCsvHelper.getPrimaryIdFromFilename(file, resKey1); + String stationId2 = GenericCsvHelper.getPrimaryIdFromFilename(file, resKey2); + + String expected = "54321"; + assertEquals(expected, stationId1); + assertEquals(expected, stationId2); + } + + @Test + public void testGetPrimaryIdFromFilename_NDBC_error_filenameTooShort() { + File file = new File("5432"); + String resKey = GenericCsvHelper.RESOURCE_KEY_NDBC_SM; + + try { + GenericCsvHelper.getPrimaryIdFromFilename(file, resKey); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Unsupported format for file:")); + } + } + + @Test + public void testGetPrimaryIdFromFilename_GBOV_success() { + File file = new File("GBOV__site--name__station--name__20021023T000000Z__20021031T235900Z.csv"); + String resKey = GenericCsvHelper.RESOURCE_KEY_GBOV; + + String siteId = GenericCsvHelper.getPrimaryIdFromFilename(file, resKey); + + String expected = "site name"; + assertEquals(expected, siteId); + } + + @Test + public void testGetPrimaryIdFromFilename_error_unsupportedFormat() { + File file = new File("54321c2017.txt"); + String resKey = "unsupported_format_key"; + + try { + GenericCsvHelper.getPrimaryIdFromFilename(file, resKey); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Unsupported format for file:")); + } + } + + @Test + public void testGetSecondaryIdFromFilename_GBOV_success() { + File file = new File("GBOV__site--name__station--name__20021023T000000Z__20021031T235900Z.csv"); + String resKey = GenericCsvHelper.RESOURCE_KEY_GBOV; + + String stationId = GenericCsvHelper.getSecondaryIdFromFilename(file, resKey); + + String expected = "station name"; + assertEquals(expected, stationId); + } + + @Test + public void testGetSecondaryIdFromFilename_noSecondaryId() { + File file = new File("someName.txt"); + + String id1 = GenericCsvHelper.getSecondaryIdFromFilename(file, null); + String id2 = GenericCsvHelper.getSecondaryIdFromFilename(file, GenericCsvHelper.RESOURCE_KEY_NDBC_CW); + String id3 = GenericCsvHelper.getSecondaryIdFromFilename(file, GenericCsvHelper.RESOURCE_KEY_NDBC_SM); + + assertNull(id1); + assertNull(id2); + assertNull(id3); + } + + @Test + public void testGetNcDataType() { + assertEquals(DataType.BYTE, GenericCsvHelper.getNcDataType("byte")); + assertEquals(DataType.SHORT, GenericCsvHelper.getNcDataType("short")); + assertEquals(DataType.INT, GenericCsvHelper.getNcDataType("int")); + assertEquals(DataType.LONG, GenericCsvHelper.getNcDataType("long")); + assertEquals(DataType.FLOAT, GenericCsvHelper.getNcDataType("float")); + assertEquals(DataType.DOUBLE, GenericCsvHelper.getNcDataType("double")); + assertEquals(DataType.STRING, GenericCsvHelper.getNcDataType("string")); + assertEquals(DataType.STRING, GenericCsvHelper.getNcDataType("lalaaa")); + } + + @Test + public void testGetFillValueClass() { + assertEquals(byte.class, GenericCsvHelper.getFillValueClass("byte")); + assertEquals(short.class, GenericCsvHelper.getFillValueClass("short")); + assertEquals(int.class, GenericCsvHelper.getFillValueClass("int")); + assertEquals(long.class, GenericCsvHelper.getFillValueClass("long")); + assertEquals(float.class, GenericCsvHelper.getFillValueClass("float")); + assertEquals(double.class, GenericCsvHelper.getFillValueClass("double")); + + try { + + assertEquals(byte.class, GenericCsvHelper.getFillValueClass("string")); + fail(); + } catch (RuntimeException ignore) {} + } + + @Test + public void testCastFillValue() { + Number num = GenericCsvHelper.castFillValue(1.0, "byte"); + assertEquals((byte) 1, num); + num = GenericCsvHelper.castFillValue(1.0, "short"); + assertEquals((short) 1, num); + num = GenericCsvHelper.castFillValue(1.0, "int"); + assertEquals(1, num); + num = GenericCsvHelper.castFillValue(1.0, "long"); + assertEquals((long) 1, num); + num = GenericCsvHelper.castFillValue(1.0, "float"); + assertEquals((float) 1, num); + num = GenericCsvHelper.castFillValue(1.0, "double"); + assertEquals(1.0, num); + num = GenericCsvHelper.castFillValue(1.0, "sdyhgsr"); + assertEquals(1.0, num); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvHelper_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvHelper_IO_Test.java new file mode 100644 index 000000000..5a0a7b6aa --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvHelper_IO_Test.java @@ -0,0 +1,249 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import com.bc.fiduceo.IOTestRunner; +import com.bc.fiduceo.TestUtil; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.*; + +@RunWith(IOTestRunner.class) +public class GenericCsvHelper_IO_Test { + + + @Test + public void test_parsingSmFile() throws IOException { + final File testFile = getSM(); + CsvFormatConfig config = CsvFormatConfig.loadConfig(GenericCsvHelper.RESOURCE_KEY_NDBC_SM); + List records = GenericCsvHelper.parseData(testFile, config, GenericCsvHelper.RESOURCE_KEY_NDBC_SM); + + assertNotNull(records); + assertEquals(41834, records.size()); + + GenericRecord record = records.get(0); + assertNotNull(record); + assertEquals(14, record.getValues().size()); + assertEquals(1483228800, record.get("time")); + assertEquals((short) 81, record.get("WDIR")); + assertEquals(6.4f, record.get("WSPD")); + assertEquals(9.5f, record.get("GST")); + assertEquals(99.0f, record.get("WVHT")); + assertEquals(99.0f, record.get("DPD")); + assertEquals(99.0f, record.get("APD")); + assertEquals((short) 999, record.get("MWD")); + assertEquals(1015.0f, record.get("PRES")); + assertEquals(27.2f, record.get("ATMP")); + assertEquals(27.3f, record.get("WTMP")); + assertEquals(20.3f, record.get("DEWP")); + assertEquals(99.0f, record.get("VIS")); + assertEquals(99.0f, record.get("TIDE")); + + record = records.get(41833); + assertNotNull(record); + assertEquals(14, record.getValues().size()); + assertEquals(1508374800, record.get("time")); + assertEquals((short) 148, record.get("WDIR")); + assertEquals(4.1f, record.get("WSPD")); + assertEquals(11.0f, record.get("GST")); + assertEquals(99.0f, record.get("WVHT")); + assertEquals(99.0f, record.get("DPD")); + assertEquals(99.0f, record.get("APD")); + assertEquals((short) 999, record.get("MWD")); + assertEquals(1014.0f, record.get("PRES")); + assertEquals(26.9f, record.get("ATMP")); + assertEquals(999.0f, record.get("WTMP")); + assertEquals(23.3f, record.get("DEWP")); + assertEquals(99.0f, record.get("VIS")); + assertEquals(99.0f, record.get("TIDE")); + } + + @Test + public void test_parsingCwFile() throws IOException { + final File testFile = getCW(); + CsvFormatConfig config = CsvFormatConfig.loadConfig(GenericCsvHelper.RESOURCE_KEY_NDBC_CW); + List records = GenericCsvHelper.parseData(testFile, config, GenericCsvHelper.RESOURCE_KEY_NDBC_CW); + + assertNotNull(records); + assertEquals(30516, records.size()); + + GenericRecord record = records.get(0); + assertNotNull(record); + assertEquals(6, record.getValues().size()); + assertEquals(1464735600, record.get("time")); + assertEquals((short) 999, record.get("WDIR")); + assertEquals(5.7f, record.get("WSPD")); + assertEquals((short) 999, record.get("GDR")); + assertEquals(99.0f, record.get("GST")); + assertEquals((short) 9999, record.get("GTIME")); + + record = records.get(30515); + assertNotNull(record); + assertEquals(6, record.getValues().size()); + assertEquals(1483224600, record.get("time")); + assertEquals((short) 184, record.get("WDIR")); + assertEquals(7.6f, record.get("WSPD")); + assertEquals((short) 179, record.get("GDR")); + assertEquals(9.8f, record.get("GST")); + assertEquals((short) 2157, record.get("GTIME")); + } + + @Test + public void test_parsingGBOVFile() throws IOException { + final File testFile = getGBOV(); + CsvFormatConfig config = CsvFormatConfig.loadConfig(GenericCsvHelper.RESOURCE_KEY_GBOV); + List records = GenericCsvHelper.parseData(testFile, config, GenericCsvHelper.RESOURCE_KEY_GBOV); + + assertNotNull(records); + assertEquals(3, records.size()); + + GenericRecord record = records.get(0); + assertNotNull(record); + assertEquals(61, record.getValues().size()); + assertEquals(1464739200, record.get("TIME_IS")); + assertEquals(0.617f, record.get("FIPAR_down")); + assertEquals(0.025f, record.get("FIPAR_down_err")); + assertEquals(-999f, record.get("FIPAR_total")); + assertEquals(-999f, record.get("FIPAR_total_err")); + assertEquals(0.959f, record.get("FIPAR_up")); + assertEquals(0.012f, record.get("FIPAR_up_err")); + assertEquals((short) 68, record.get("RM6_down_flag")); + assertEquals((short) 8, record.get("RM6_up_flag")); + assertEquals(-999f, record.get("Clumping_Miller")); + assertEquals(-999f, record.get("Clumping_Miller_err")); + assertEquals(-999f, record.get("Clumping_Warren")); + assertEquals(-999f, record.get("Clumping_Warren_err")); + assertEquals(1.52f, record.get("LAI_Miller_down")); + assertEquals(0.05f, record.get("LAI_Miller_down_err")); + assertEquals(6.14f, record.get("LAI_Miller_up")); + assertEquals(0.15f, record.get("LAI_Miller_up_err")); + assertEquals(1.2f, record.get("LAI_Warren_down")); + assertEquals(0.06f, record.get("LAI_Warren_down_err")); + assertEquals(4.61f, record.get("LAI_Warren_up")); + assertEquals(.15f, record.get("LAI_Warren_up_err")); + assertEquals(-999f, record.get("LAI_down")); + assertEquals(-999f, record.get("LAI_total_Miller")); + assertEquals(-999f, record.get("LAI_total_Warren")); + assertEquals(1.41f, record.get("LAIe_Miller_down")); + assertEquals(.04f, record.get("LAIe_Miller_down_err")); + assertEquals(4.95f, record.get("LAIe_Miller_up")); + assertEquals(.13f, record.get("LAIe_Miller_up_err")); + assertEquals(1.1f, record.get("LAIe_Warren_down")); + assertEquals(.04f, record.get("LAIe_Warren_down_err")); + assertEquals(3.63f, record.get("LAIe_Warren_up")); + assertEquals(.14f, record.get("LAIe_Warren_up_err")); + assertEquals(-999f, record.get("PAI_Miller")); + assertEquals(-999f, record.get("PAI_Miller_err")); + assertEquals(-999f, record.get("PAI_Warren")); + assertEquals(-999f, record.get("PAI_Warren_err")); + assertEquals(-999f, record.get("PAIe_Miller")); + assertEquals(-999f, record.get("PAIe_Miller_err")); + assertEquals(-999f, record.get("PAIe_Warren")); + assertEquals(-999f, record.get("PAIe_Warren_err")); + assertEquals(.93f, record.get("clumping_Miller_down")); + assertEquals(.04f, record.get("clumping_Miller_down_err")); + assertEquals(.805f , record.get("clumping_Miller_up")); + assertEquals(.029f, record.get("clumping_Miller_up_err")); + assertEquals(.92f, record.get("clumping_Warren_down")); + assertEquals(.06f, record.get("clumping_Warren_down_err")); + assertEquals(.79f, record.get("clumping_Warren_up")); + assertEquals(.04f, record.get("clumping_Warren_up_err")); + assertEquals((short) 68, record.get("RM7_down_flag")); + assertEquals((short) 8, record.get("RM7_up_flag")); + assertEquals(-999f, record.get("LSE")); + assertEquals(-999f, record.get("LSE_STD")); + assertEquals(-999f, record.get("LSR")); + assertEquals(-999f, record.get("LSR_STD")); + assertEquals((short) -999, record.get("QC_LSE")); + assertEquals((short) -999, record.get("QC_LSR")); + assertEquals(-999f, record.get("LST")); + assertEquals(-999f, record.get("LST_STD")); + assertEquals((short) -999, record.get("QC_LST")); + assertEquals((short) -999, record.get("QC_SM_5")); + assertEquals(-999.0, record.get("SM_5")); + + + record = records.get(2); + assertNotNull(record); + assertEquals(61, record.getValues().size()); + assertEquals(1467072000, record.get("TIME_IS")); + assertEquals(0.443f, record.get("FIPAR_down")); + assertEquals(0.025f, record.get("FIPAR_down_err")); + assertEquals(-999f, record.get("FIPAR_total")); + assertEquals(-999f, record.get("FIPAR_total_err")); + assertEquals(0.931f, record.get("FIPAR_up")); + assertEquals(0.021f, record.get("FIPAR_up_err")); + assertEquals((short) 8, record.get("RM6_down_flag")); + assertEquals((short) 8, record.get("RM6_up_flag")); + assertEquals(-999f, record.get("Clumping_Miller")); + assertEquals(-999f, record.get("Clumping_Miller_err")); + assertEquals(-999f, record.get("Clumping_Warren")); + assertEquals(-999f, record.get("Clumping_Warren_err")); + assertEquals(.96f, record.get("LAI_Miller_down")); + assertEquals(.034f, record.get("LAI_Miller_down_err")); + assertEquals(5.53f, record.get("LAI_Miller_up")); + assertEquals(.23f, record.get("LAI_Miller_up_err")); + assertEquals(.78f, record.get("LAI_Warren_down")); + assertEquals(0.05f, record.get("LAI_Warren_down_err")); + assertEquals(4.2f, record.get("LAI_Warren_up")); + assertEquals(.4f, record.get("LAI_Warren_up_err")); + assertEquals(-999f, record.get("LAI_down")); + assertEquals(-999f, record.get("LAI_total_Miller")); + assertEquals(-999f, record.get("LAI_total_Warren")); + assertEquals(.898f, record.get("LAIe_Miller_down")); + assertEquals(.03f, record.get("LAIe_Miller_down_err")); + assertEquals(4.05f, record.get("LAIe_Miller_up")); + assertEquals(.22f, record.get("LAIe_Miller_up_err")); + assertEquals(.73f, record.get("LAIe_Warren_down")); + assertEquals(.04f, record.get("LAIe_Warren_down_err")); + assertEquals(2.9f, record.get("LAIe_Warren_up")); + assertEquals(.4f, record.get("LAIe_Warren_up_err")); + assertEquals(-999f, record.get("PAI_Miller")); + assertEquals(-999f, record.get("PAI_Miller_err")); + assertEquals(-999f, record.get("PAI_Warren")); + assertEquals(-999f, record.get("PAI_Warren_err")); + assertEquals(-999f, record.get("PAIe_Miller")); + assertEquals(-999f, record.get("PAIe_Miller_err")); + assertEquals(-999f, record.get("PAIe_Warren")); + assertEquals(-999f, record.get("PAIe_Warren_err")); + assertEquals(.94f, record.get("clumping_Miller_down")); + assertEquals(.05f, record.get("clumping_Miller_down_err")); + assertEquals(.73f , record.get("clumping_Miller_up")); + assertEquals(.05f, record.get("clumping_Miller_up_err")); + assertEquals(.94f, record.get("clumping_Warren_down")); + assertEquals(.07f, record.get("clumping_Warren_down_err")); + assertEquals(.7f, record.get("clumping_Warren_up")); + assertEquals(.12f, record.get("clumping_Warren_up_err")); + assertEquals((short) 8, record.get("RM7_down_flag")); + assertEquals((short) 8, record.get("RM7_up_flag")); + assertEquals(-999f, record.get("LSE")); + assertEquals(-999f, record.get("LSE_STD")); + assertEquals(-999f, record.get("LSR")); + assertEquals(-999f, record.get("LSR_STD")); + assertEquals((short) -999, record.get("QC_LSE")); + assertEquals((short) -999, record.get("QC_LSR")); + assertEquals(-999f, record.get("LST")); + assertEquals(-999f, record.get("LST_STD")); + assertEquals((short) -999, record.get("QC_LST")); + assertEquals((short) -999, record.get("QC_SM_5")); + assertEquals(-999.0, record.get("SM_5")); + } + + private static File getSM() throws IOException { + final String relativePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "ndbc", "ndbc-sm-cb", "v1", "2017", "42088h2017.txt"}, false); + return TestUtil.getTestDataFileAsserted(relativePath); + } + + private static File getCW() throws IOException { + final String relativePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "ndbc", "ndbc-cw-ob", "v1", "2016", "42002c2016.txt"}, false); + return TestUtil.getTestDataFileAsserted(relativePath); + } + + private static File getGBOV() throws IOException { + final String relativePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "gbov", "v1", "2016", "06", "GBOV__Bartlett--Experimental--Forest__BART_047__20160601T000000Z__20160628T000000Z.csv"}, false); + return TestUtil.getTestDataFileAsserted(relativePath); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader_GBOV_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader_GBOV_IO_Test.java new file mode 100644 index 000000000..8cdc7ed18 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader_GBOV_IO_Test.java @@ -0,0 +1,829 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import com.bc.fiduceo.IOTestRunner; +import com.bc.fiduceo.NCTestUtils; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.core.NodeType; +import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.netcdf.StringVariable; +import com.bc.fiduceo.reader.time.TimeLocator; +import com.bc.fiduceo.util.VariableProxy; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.Array; +import ucar.ma2.ArrayInt; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Variable; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.*; + +@RunWith(IOTestRunner.class) +public class GenericCsvReader_GBOV_IO_Test { + + private GenericCsvReader reader; + + @Before + public void setUp() { + reader = new GenericCsvReader(GenericCsvHelper.RESOURCE_KEY_GBOV); + } + + @Test + public void testReadAcquisitionInfo() throws IOException { + final File testFile = getGBOV(); + + try { + reader.open(testFile); + final AcquisitionInfo info = reader.read(); + + TestUtil.assertCorrectUTCDate(2016, 6, 1, 0, 0, 0, 0, info.getSensingStart()); + TestUtil.assertCorrectUTCDate(2016, 6, 28, 0, 0, 0, 0, info.getSensingStop()); + + assertEquals(NodeType.UNDEFINED, info.getNodeType()); + + assertNull(info.getBoundingGeometry()); + assertNull(info.getTimeAxes()); + } finally { + reader.close(); + } + } + + @Test + public void testPixelLocatorNotImplemented() throws IOException { + final File testFile = getGBOV(); + + try { + reader.open(testFile); + reader.getPixelLocator(); + fail(); + } catch (RuntimeException ignore) { + } finally { + reader.close(); + } + } + + @Test + public void testSubscenePixelLocatorNotImplemented() throws IOException { + final File testFile = getGBOV(); + + try { + reader.open(testFile); + reader.getSubScenePixelLocator(null); + fail(); + } catch (RuntimeException ignore) { + } finally { + reader.close(); + } + } + + @Test + public void testGetTimeLocator() throws IOException { + final File testFile = getGBOV(); + + try { + reader.open(testFile); + final TimeLocator timeLocator = reader.getTimeLocator(); + + assertNotNull(timeLocator); + assertEquals(1464739200000L, timeLocator.getTimeFor(0, 0)); + assertEquals(1465862400000L, timeLocator.getTimeFor(0, 1)); + assertEquals(1467072000000L, timeLocator.getTimeFor(0, 2)); + + } finally { + reader.close(); + } + } + + @Test + public void testExtractYearMonthDayFromFilename_GBOV_success() throws IOException { + final File testFile = getGBOV(); + + try { + reader.open(testFile); + int[] ymd = reader.extractYearMonthDayFromFilename(testFile.getAbsolutePath()); + + assertEquals(3, ymd.length); + assertEquals(2016, ymd[0]); + assertEquals(6, ymd[1]); + assertEquals(1, ymd[2]); + } finally { + reader.close(); + } + } + + @Test + public void testReadRaw() throws InvalidRangeException,IOException { + final File testFile = getGBOV(); + + try { + reader.open(testFile); + + Array array = reader.readRaw(7, 0, new Interval(1, 1), "TIME_IS"); + assertEquals(DataType.INT, array.getDataType()); + assertEquals(1464739200, array.getInt(0)); + + array = reader.readRaw(7, 2004, new Interval(1, 1), "site"); + assertEquals(DataType.STRING, array.getDataType()); + assertEquals("Bartlett Experimental Forest", array.getObject(0)); + + array = reader.readRaw(7, 2004, new Interval(1, 1), "station"); + assertEquals(DataType.STRING, array.getDataType()); + assertEquals("BART_047", array.getObject(0)); + + array = reader.readRaw(7, 2004, new Interval(1, 1), "IGBP_class"); + assertEquals(DataType.STRING, array.getDataType()); + assertEquals("Mixed Forest", array.getObject(0)); + + array = reader.readRaw(12, 5, new Interval(1, 1), "elevation"); + assertEquals(232, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "Lat_IS"); + assertEquals(44.063901f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "Lon_IS"); + assertEquals(-71.287308f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + + array = reader.readRaw(12, 0, new Interval(1, 1), "FIPAR_down"); + assertEquals(0.617f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "FIPAR_down_err"); + assertEquals(0.025f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "FIPAR_total"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "FIPAR_total_err"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "FIPAR_up"); + assertEquals(0.959f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "FIPAR_up_err"); + assertEquals(0.012f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "Clumping_Miller"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "Clumping_Miller_err"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "Clumping_Warren"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "Clumping_Warren_err"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAI_Miller_down"); + assertEquals(1.52f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAI_Miller_down_err"); + assertEquals(0.05f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAI_Miller_up"); + assertEquals(6.14f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAI_Miller_up_err"); + assertEquals(0.15f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAI_Warren_down"); + assertEquals(1.2f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAI_Warren_down_err"); + assertEquals(.06f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAI_Warren_up"); + assertEquals(4.61f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAI_Warren_up_err"); + assertEquals(.15f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAI_down"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAI_total_Miller"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAI_total_Warren"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAIe_Miller_down"); + assertEquals(1.41f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAIe_Miller_down_err"); + assertEquals(.04f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAIe_Miller_up"); + assertEquals(4.95f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAIe_Miller_up_err"); + assertEquals(.13f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAIe_Warren_down"); + assertEquals(1.1f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAIe_Warren_down_err"); + assertEquals(.04f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAIe_Warren_up"); + assertEquals(3.63f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LAIe_Warren_up_err"); + assertEquals(.14f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "PAI_Miller"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "PAI_Miller_err"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "PAI_Warren"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "PAI_Warren_err"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "PAIe_Miller"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "PAIe_Miller_err"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "PAIe_Warren"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "PAIe_Warren_err"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "clumping_Miller_down"); + assertEquals(.93f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "clumping_Miller_down_err"); + assertEquals(.04f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "clumping_Miller_up"); + assertEquals(.805f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "clumping_Miller_up_err"); + assertEquals(.029f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "clumping_Warren_down"); + assertEquals(.92f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "clumping_Warren_down_err"); + assertEquals(.06f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "clumping_Warren_up"); + assertEquals(.79f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "clumping_Warren_up_err"); + assertEquals(.04f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LSE"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LSE_STD"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LSR"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LSR_STD"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LST"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "LST_STD"); + assertEquals(-999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 0, new Interval(1, 1), "SM_5"); + assertEquals(-999., array.getDouble(0), 1e-8); + assertEquals(DataType.DOUBLE, array.getDataType()); + + + array = reader.readRaw(12, 2, new Interval(1, 1), "RM6_down_flag"); + assertEquals((short) 8, array.getShort(0)); + assertEquals(DataType.SHORT, array.getDataType()); + + array = reader.readRaw(12, 2, new Interval(1, 1), "RM6_up_flag"); + assertEquals((short) 8, array.getShort(0)); + assertEquals(DataType.SHORT, array.getDataType()); + + array = reader.readRaw(12, 2, new Interval(1, 1), "RM7_down_flag"); + assertEquals((short) 8, array.getShort(0)); + assertEquals(DataType.SHORT, array.getDataType()); + + array = reader.readRaw(12, 2, new Interval(1, 1), "RM7_up_flag"); + assertEquals((short) 8, array.getShort(0)); + assertEquals(DataType.SHORT, array.getDataType()); + + array = reader.readRaw(12, 2, new Interval(1, 1), "QC_LSE"); + assertEquals((short) -999, array.getShort(0)); + assertEquals(DataType.SHORT, array.getDataType()); + + array = reader.readRaw(12, 2, new Interval(1, 1), "QC_LSR"); + assertEquals((short) -999, array.getShort(0)); + assertEquals(DataType.SHORT, array.getDataType()); + + array = reader.readRaw(12, 2, new Interval(1, 1), "QC_LST"); + assertEquals((short) -999, array.getShort(0)); + assertEquals(DataType.SHORT, array.getDataType()); + + array = reader.readRaw(12, 2, new Interval(1, 1), "QC_SM_5"); + assertEquals((short) -999, array.getShort(0)); + assertEquals(DataType.SHORT, array.getDataType()); + + + } finally { + reader.close(); + } + } + + @Test + public void testReadScaled_noVarFound() throws InvalidRangeException,IOException { + final File testFile = getGBOV(); + + try { + reader.open(testFile); + + Array array = reader.readScaled(7, 0, new Interval(1, 1), "XX"); + assertNull(array); + } finally { + reader.close(); + } + } + + @Test + public void testReadAcquisitionTime() throws IOException, InvalidRangeException { + final File testFile = getGBOV(); + + try { + reader.open(testFile); + + final ArrayInt.D2 acquisitionTime = reader.readAcquisitionTime(13, 2, new Interval(1, 1)); + NCTestUtils.assertValueAt(1467072000, 0, 0, acquisitionTime); + } finally { + reader.close(); + } + } + + @Test + public void testGetVariables() throws InvalidRangeException, IOException { + final File testFile = getGBOV(); + + try { + reader.open(testFile); + + List vars = reader.getVariables(); + + assertNotNull(vars); + assertFalse(vars.isEmpty()); + assertEquals(67, vars.size()); + + StringVariable stringVar = (StringVariable) vars.get(0); + assertEquals("site", stringVar.getShortName()); + assertEquals(DataType.STRING, stringVar.getDataType()); + assertEquals(2, stringVar.getAttributes().size()); + + stringVar = (StringVariable) vars.get(1); + assertEquals("station", stringVar.getShortName()); + assertEquals(DataType.STRING, stringVar.getDataType()); + assertEquals(3, stringVar.getAttributes().size()); + + VariableProxy var = (VariableProxy) vars.get(2); + assertEquals("elevation", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + stringVar = (StringVariable) vars.get(3); + assertEquals("IGBP_class", stringVar.getShortName()); + assertEquals(DataType.STRING, stringVar.getDataType()); + assertEquals(2, stringVar.getAttributes().size()); + + var = (VariableProxy) vars.get(4); + assertEquals("Lat_IS", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(5); + assertEquals("Lon_IS", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + + var = (VariableProxy) vars.get(6); + assertEquals("TIME_IS", var.getFullName()); + assertEquals(DataType.INT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + + var = (VariableProxy) vars.get(7); + assertEquals("FIPAR_down", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(8); + assertEquals("FIPAR_down_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(9); + assertEquals("FIPAR_total", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(10); + assertEquals("FIPAR_total_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(11); + assertEquals("FIPAR_up", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(12); + assertEquals("FIPAR_up_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(13); + assertEquals("RM6_down_flag", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(14); + assertEquals("RM6_up_flag", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + + var = (VariableProxy) vars.get(15); + assertEquals("Clumping_Miller", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(16); + assertEquals("Clumping_Miller_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(17); + assertEquals("Clumping_Warren", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(18); + assertEquals("Clumping_Warren_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(19); + assertEquals("LAI_Miller_down", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(20); + assertEquals("LAI_Miller_down_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(21); + assertEquals("LAI_Miller_up", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(22); + assertEquals("LAI_Miller_up_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(23); + assertEquals("LAI_Warren_down", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(24); + assertEquals("LAI_Warren_down_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(25); + assertEquals("LAI_Warren_up", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(26); + assertEquals("LAI_Warren_up_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(27); + assertEquals("LAI_down", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(28); + assertEquals("LAI_total_Miller", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(29); + assertEquals("LAI_total_Warren", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(30); + assertEquals("LAIe_Miller_down", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(31); + assertEquals("LAIe_Miller_down_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(32); + assertEquals("LAIe_Miller_up", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(33); + assertEquals("LAIe_Miller_up_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(34); + assertEquals("LAIe_Warren_down", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(35); + assertEquals("LAIe_Warren_down_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(36); + assertEquals("LAIe_Warren_up", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(37); + assertEquals("LAIe_Warren_up_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(38); + assertEquals("PAI_Miller", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(39); + assertEquals("PAI_Miller_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(40); + assertEquals("PAI_Warren", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(41); + assertEquals("PAI_Warren_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(42); + assertEquals("PAIe_Miller", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(43); + assertEquals("PAIe_Miller_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(44); + assertEquals("PAIe_Warren", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(45); + assertEquals("PAIe_Warren_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(46); + assertEquals("clumping_Miller_down", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(47); + assertEquals("clumping_Miller_down_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(48); + assertEquals("clumping_Miller_up", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(49); + assertEquals("clumping_Miller_up_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(50); + assertEquals("clumping_Warren_down", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(51); + assertEquals("clumping_Warren_down_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(52); + assertEquals("clumping_Warren_up", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(53); + assertEquals("clumping_Warren_up_err", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(54); + assertEquals("RM7_down_flag", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(55); + assertEquals("RM7_up_flag", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + + var = (VariableProxy) vars.get(56); + assertEquals("LSE", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(57); + assertEquals("LSE_STD", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(58); + assertEquals("LSR", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(59); + assertEquals("LSR_STD", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(60); + assertEquals("QC_LSE", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(61); + assertEquals("QC_LSR", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + + var = (VariableProxy) vars.get(62); + assertEquals("LST", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(63); + assertEquals("LST_STD", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(64); + assertEquals("QC_LST", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + + var = (VariableProxy) vars.get(65); + assertEquals("QC_SM_5", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(66); + assertEquals("SM_5", var.getFullName()); + assertEquals(DataType.DOUBLE, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + + } finally { + reader.close(); + } + } + + @Test + public void testGetProductSize() throws IOException { + final File testFile = getGBOV(); + + try { + reader.open(testFile); + Dimension productSize = reader.getProductSize(); + assertEquals(3, productSize.getNy()); + } finally { + reader.close(); + } + } + + @Test + public void test_getRegex_GBOV() { + assertEquals("^GBOV__(.*?)__(.*?)__([0-9]{8}T[0-9]{6}Z)__([0-9]{8}T[0-9]{6}Z)\\.csv$", reader.getRegEx()); + } + + @Test + public void testCoordinateVariableNames() throws IOException { + final File testFile = getGBOV(); + + try { + reader.open(testFile); + String longitudeVariableName = reader.getLongitudeVariableName(); + String latitudeVariableName = reader.getLatitudeVariableName(); + assertEquals("Lon_IS", longitudeVariableName); + assertEquals("Lat_IS", latitudeVariableName); + } finally { + reader.close(); + } + } + + private static File getGBOV() throws IOException { + final String relativePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "gbov", "v1", "2016", "06", "GBOV__Bartlett--Experimental--Forest__BART_047__20160601T000000Z__20160628T000000Z.csv"}, false); + return TestUtil.getTestDataFileAsserted(relativePath); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader_NDBC_CW_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader_NDBC_CW_IO_Test.java new file mode 100644 index 000000000..3414137a4 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader_NDBC_CW_IO_Test.java @@ -0,0 +1,290 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import com.bc.fiduceo.IOTestRunner; +import com.bc.fiduceo.NCTestUtils; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.core.NodeType; +import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.netcdf.StringVariable; +import com.bc.fiduceo.reader.time.TimeLocator; +import com.bc.fiduceo.util.VariableProxy; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.Array; +import ucar.ma2.ArrayInt; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Variable; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.*; + +@RunWith(IOTestRunner.class) +public class GenericCsvReader_NDBC_CW_IO_Test { + + private GenericCsvReader reader; + + @Before + public void setUp() { + reader = new GenericCsvReader(GenericCsvHelper.RESOURCE_KEY_NDBC_CW); + } + + @Test + public void testReadAcquisitionInfo() throws IOException { + final File testFile = getCW(); + + try { + reader.open(testFile); + final AcquisitionInfo info = reader.read(); + + TestUtil.assertCorrectUTCDate(2016, 5, 31, 23, 0, 0, 0, info.getSensingStart()); + TestUtil.assertCorrectUTCDate(2016, 12, 31, 22, 50, 0, 0, info.getSensingStop()); + + assertEquals(NodeType.UNDEFINED, info.getNodeType()); + + assertNull(info.getBoundingGeometry()); + assertNull(info.getTimeAxes()); + } finally { + reader.close(); + } + } + + @Test + public void testGetTimeLocator() throws IOException { + final File testFile = getCW(); + + try { + reader.open(testFile); + final TimeLocator timeLocator = reader.getTimeLocator(); + + assertNotNull(timeLocator); + assertEquals(1464735600000L, timeLocator.getTimeFor(0, 0)); + assertEquals(1473843600000L, timeLocator.getTimeFor(0, 15000)); + assertEquals(1483224600000L, timeLocator.getTimeFor(0, 30515)); + + } finally { + reader.close(); + } + } + + @Test + public void testExtractYearMonthDayFromFilename_GBOV_success() throws IOException { + final File testFile = getCW(); + + try { + reader.open(testFile); + int[] ymd = reader.extractYearMonthDayFromFilename(testFile.getAbsolutePath()); + + assertEquals(3, ymd.length); + assertEquals(2016, ymd[0]); + assertEquals(1, ymd[1]); + assertEquals(1, ymd[2]); + } finally { + reader.close(); + } + } + + @Test + public void testReadRaw() throws InvalidRangeException,IOException { + final File testFile = getCW(); + + try { + reader.open(testFile); + + Array array = reader.readRaw(7, 2004, new Interval(1, 1), "id"); + assertEquals(DataType.STRING, array.getDataType()); + assertEquals("42002", array.getObject(0)); + + array = reader.readRaw(7, 4, new Interval(1, 1), "station_type"); + assertEquals(DataType.BYTE, array.getDataType()); + assertEquals(0, array.getByte(0)); + + array = reader.readRaw(7, 4, new Interval(1, 1), "measurement_type"); + assertEquals(DataType.BYTE, array.getDataType()); + assertEquals(0, array.getByte(0)); + + array = reader.readRaw(8, 5, new Interval(1, 1), "latitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(26.055f, array.getFloat(0), 1e-8); + + array = reader.readRaw(8, 5, new Interval(1, 1), "longitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-93.646f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 6, new Interval(1, 1), "anemometer_height"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(3.8f, array.getFloat(0), 1e-8); + + + array = reader.readRaw(11, 8, new Interval(1, 1), "WDIR"); + assertEquals(DataType.SHORT, array.getDataType()); + assertEquals((short) 999, array.getShort(0)); + + array = reader.readRaw(11, 8, new Interval(1, 1), "WSPD"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(7.2f, array.getFloat(0), 1e-8); + + array = reader.readRaw(12, 9, new Interval(1, 1), "GDR"); + assertEquals((short) 999, array.getShort(0)); + assertEquals(DataType.SHORT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "GST"); + assertEquals(7.6f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 23, new Interval(1, 1), "GTIME"); + assertEquals((short) 246, array.getShort(0)); + assertEquals(DataType.SHORT, array.getDataType()); + + array = reader.readRaw(7, 0, new Interval(1, 1), "time"); + assertEquals(DataType.INT, array.getDataType()); + assertEquals(1464735600, array.getInt(0)); + + } finally { + reader.close(); + } + } + + @Test + public void testReadAcquisitionTime() throws IOException, InvalidRangeException { + final File testFile = getCW(); + + try { + reader.open(testFile); + + final ArrayInt.D2 acquisitionTime = reader.readAcquisitionTime(13, 30515, new Interval(1, 1)); + NCTestUtils.assertValueAt(1483224600, 0, 0, acquisitionTime); + } finally { + reader.close(); + } + } + + @Test + public void testGetVariables() throws InvalidRangeException, IOException { + final File testFile = getCW(); + + try { + reader.open(testFile); + + List vars = reader.getVariables(); + + assertNotNull(vars); + assertFalse(vars.isEmpty()); + assertEquals(12, vars.size()); + + StringVariable stringVar = (StringVariable) vars.get(0); + assertEquals("id", stringVar.getShortName()); + assertEquals(DataType.STRING, stringVar.getDataType()); + assertEquals(3, stringVar.getAttributes().size()); + + VariableProxy var = (VariableProxy) vars.get(1); + assertEquals("latitude", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(2); + assertEquals("longitude", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(3); + assertEquals("station_type", var.getFullName()); + assertEquals(DataType.BYTE, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(4); + assertEquals("measurement_type", var.getFullName()); + assertEquals(DataType.BYTE, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(5); + assertEquals("anemometer_height", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(6); + assertEquals("time", var.getFullName()); + assertEquals(DataType.INT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(7); + assertEquals("WDIR", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(8); + assertEquals("WSPD", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(9); + assertEquals("GDR", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(10); + assertEquals("GST", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(11); + assertEquals("GTIME", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + } finally { + reader.close(); + } + } + + @Test + public void testGetProductSize() throws IOException { + final File testFile = getCW(); + + try { + reader.open(testFile); + Dimension productSize = reader.getProductSize(); + assertEquals(30516, productSize.getNy()); + } finally { + reader.close(); + } + } + + @Test + public void test_getRegex_NDBC_CW() throws IOException { + final File testFile = getCW(); + + try { + reader.open(testFile); + assertEquals("\\w{5}c\\d{4}.txt", reader.getRegEx()); + } finally { + reader.close(); + } + } + + @Test + public void testCoordinateVariableNames() throws IOException { + final File testFile = getCW(); + + try { + reader.open(testFile); + String longitudeVariableName = reader.getLongitudeVariableName(); + String latitudeVariableName = reader.getLatitudeVariableName(); + assertEquals("longitude", longitudeVariableName); + assertEquals("latitude", latitudeVariableName); + } finally { + reader.close(); + } + } + + private static File getCW() throws IOException { + final String relativePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "ndbc", "ndbc-cw-ob", "v1", "2016", "42002c2016.txt"}, false); + return TestUtil.getTestDataFileAsserted(relativePath); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader_NDBC_SM_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader_NDBC_SM_IO_Test.java new file mode 100644 index 000000000..3dbc63d0f --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericCsvReader_NDBC_SM_IO_Test.java @@ -0,0 +1,392 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import com.bc.fiduceo.IOTestRunner; +import com.bc.fiduceo.NCTestUtils; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.core.NodeType; +import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.netcdf.StringVariable; +import com.bc.fiduceo.reader.time.TimeLocator; +import com.bc.fiduceo.util.VariableProxy; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.Array; +import ucar.ma2.ArrayInt; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Variable; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertNull; + +@RunWith(IOTestRunner.class) +public class GenericCsvReader_NDBC_SM_IO_Test { + + private GenericCsvReader reader; + + @Before + public void setUp() { + reader = new GenericCsvReader(GenericCsvHelper.RESOURCE_KEY_NDBC_SM); + } + + @Test + public void testReadAcquisitionInfo() throws IOException { + final File testFile = getSM(); + + try { + reader.open(testFile); + final AcquisitionInfo info = reader.read(); + + TestUtil.assertCorrectUTCDate(2017, 1, 1, 0, 0, 0, 0, info.getSensingStart()); + TestUtil.assertCorrectUTCDate(2017, 10, 19, 1, 0, 0, 0, info.getSensingStop()); + + assertEquals(NodeType.UNDEFINED, info.getNodeType()); + + assertNull(info.getBoundingGeometry()); + assertNull(info.getTimeAxes()); + } finally { + reader.close(); + } + } + + @Test + public void testGetTimeLocator() throws IOException { + final File testFile = getSM(); + + try { + reader.open(testFile); + final TimeLocator timeLocator = reader.getTimeLocator(); + + assertNotNull(timeLocator); + assertEquals(1483228800000L, timeLocator.getTimeFor(0, 0)); + assertEquals(1496464200000L, timeLocator.getTimeFor(0, 22000)); + assertEquals(1508374800000L, timeLocator.getTimeFor(0, 41833)); + + } finally { + reader.close(); + } + } + + @Test + public void testExtractYearMonthDayFromFilename_GBOV_success() throws IOException { + final File testFile = getSM(); + + try { + reader.open(testFile); + int[] ymd = reader.extractYearMonthDayFromFilename(testFile.getAbsolutePath()); + + assertEquals(3, ymd.length); + assertEquals(2017, ymd[0]); + assertEquals(1, ymd[1]); + assertEquals(1, ymd[2]); + } finally { + reader.close(); + } + } + + @Test + public void testReadRaw() throws InvalidRangeException,IOException { + final File testFile = getSM(); + + try { + reader.open(testFile); + + Array array = reader.readRaw(7, 2004, new Interval(1, 1), "id"); + assertEquals(DataType.STRING, array.getDataType()); + assertEquals("42088", array.getObject(0)); + + array = reader.readRaw(7, 4, new Interval(1, 1), "station_type"); + assertEquals(DataType.BYTE, array.getDataType()); + assertEquals(1, array.getByte(0)); + + array = reader.readRaw(7, 4, new Interval(1, 1), "measurement_type"); + assertEquals(DataType.BYTE, array.getDataType()); + assertEquals(1, array.getByte(0)); + + array = reader.readRaw(8, 5, new Interval(1, 1), "latitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(11.301f, array.getFloat(0), 1e-8); + + array = reader.readRaw(8, 5, new Interval(1, 1), "longitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-60.521f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 6, new Interval(1, 1), "anemometer_height"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(3.35f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 6, new Interval(1, 1), "air_temp_height"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(Float.NaN, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 6, new Interval(1, 1), "barometer_height"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(Float.NaN, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 6, new Interval(1, 1), "sst_depth"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(Float.NaN, array.getFloat(0), 1e-8); + + + array = reader.readRaw(11, 8, new Interval(1, 1), "WDIR"); + assertEquals(DataType.SHORT, array.getDataType()); + assertEquals((short) 83, array.getShort(0)); + + array = reader.readRaw(11, 8, new Interval(1, 1), "WSPD"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(6.0f, array.getFloat(0), 1e-8); + + array = reader.readRaw(12, 5, new Interval(1, 1), "GST"); + assertEquals(8.3f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "WVHT"); + assertEquals(99.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "DPD"); + assertEquals(99.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "APD"); + assertEquals(99.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 8, new Interval(1, 1), "MWD"); + assertEquals((short) 999, array.getShort(0)); + assertEquals(DataType.SHORT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "PRES"); + assertEquals(1015.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "ATMP"); + assertEquals(27.2f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "WTMP"); + assertEquals(999.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "DEWP"); + assertEquals(20.2f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "VIS"); + assertEquals(99.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(12, 5, new Interval(1, 1), "TIDE"); + assertEquals(99.f, array.getFloat(0), 1e-8); + assertEquals(DataType.FLOAT, array.getDataType()); + + array = reader.readRaw(7, 0, new Interval(1, 1), "time"); + assertEquals(DataType.INT, array.getDataType()); + assertEquals(1483228800, array.getInt(0)); + + } finally { + reader.close(); + } + } + + @Test + public void testReadAcquisitionTime() throws IOException, InvalidRangeException { + final File testFile = getSM(); + + try { + reader.open(testFile); + + final ArrayInt.D2 acquisitionTime = reader.readAcquisitionTime(13, 41833, new Interval(1, 1)); + NCTestUtils.assertValueAt(1508374800, 0, 0, acquisitionTime); + } finally { + reader.close(); + } + } + + @Test + public void testGetVariables() throws InvalidRangeException, IOException { + final File testFile = getSM(); + + try { + reader.open(testFile); + + List vars = reader.getVariables(); + + assertNotNull(vars); + assertFalse(vars.isEmpty()); + assertEquals(23, vars.size()); + + StringVariable stringVar = (StringVariable) vars.get(0); + assertEquals("id", stringVar.getShortName()); + assertEquals(DataType.STRING, stringVar.getDataType()); + assertEquals(3, stringVar.getAttributes().size()); + + VariableProxy var = (VariableProxy) vars.get(1); + assertEquals("latitude", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(2); + assertEquals("longitude", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(3); + assertEquals("station_type", var.getFullName()); + assertEquals(DataType.BYTE, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(4); + assertEquals("measurement_type", var.getFullName()); + assertEquals(DataType.BYTE, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(5); + assertEquals("anemometer_height", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(6); + assertEquals("air_temp_height", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(7); + assertEquals("barometer_height", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(8); + assertEquals("sst_depth", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(9); + assertEquals("time", var.getFullName()); + assertEquals(DataType.INT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + + var = (VariableProxy) vars.get(10); + assertEquals("WDIR", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(11); + assertEquals("WSPD", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(12); + assertEquals("GST", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(13); + assertEquals("WVHT", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(14); + assertEquals("DPD", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(15); + assertEquals("APD", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(16); + assertEquals("MWD", var.getFullName()); + assertEquals(DataType.SHORT, var.getDataType()); + assertEquals(3, var.getAttributes().size()); + + var = (VariableProxy) vars.get(17); + assertEquals("PRES", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(18); + assertEquals("ATMP", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(19); + assertEquals("WTMP", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(20); + assertEquals("DEWP", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(21); + assertEquals("VIS", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + var = (VariableProxy) vars.get(22); + assertEquals("TIDE", var.getFullName()); + assertEquals(DataType.FLOAT, var.getDataType()); + assertEquals(4, var.getAttributes().size()); + + + } finally { + reader.close(); + } + } + + @Test + public void testGetProductSize() throws IOException { + final File testFile = getSM(); + + try { + reader.open(testFile); + Dimension productSize = reader.getProductSize(); + assertEquals(41834, productSize.getNy()); + } finally { + reader.close(); + } + } + + @Test + public void test_getRegex_NDBC_SM() throws IOException { + final File testFile = getSM(); + + try { + reader.open(testFile); + assertEquals("\\w{5}h\\d{4}.txt", reader.getRegEx()); + } finally { + reader.close(); + } + } + + @Test + public void testCoordinateVariableNames() throws IOException { + final File testFile = getSM(); + + try { + reader.open(testFile); + String longitudeVariableName = reader.getLongitudeVariableName(); + String latitudeVariableName = reader.getLatitudeVariableName(); + assertEquals("longitude", longitudeVariableName); + assertEquals("latitude", latitudeVariableName); + } finally { + reader.close(); + } + } + + private static File getSM() throws IOException { + final String relativePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "ndbc", "ndbc-sm-cb", "v1", "2017", "42088h2017.txt"}, false); + return TestUtil.getTestDataFileAsserted(relativePath); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericVariableTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericVariableTest.java new file mode 100644 index 000000000..988d79225 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/GenericVariableTest.java @@ -0,0 +1,70 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import org.esa.snap.core.datamodel.ProductData; +import org.junit.Test; + +import static org.junit.Assert.*; + + +public class GenericVariableTest { + + @Test + public void setProductData_byte() { + GenericVariable var = new GenericVariable(); + var.setType("byte"); + assertEquals(ProductData.TYPE_INT8, var.getProductData()); + } + + @Test + public void setProductData_short() { + GenericVariable var = new GenericVariable(); + var.setType("short"); + assertEquals(ProductData.TYPE_INT16, var.getProductData()); + } + + @Test + public void setProductData_int() { + GenericVariable var = new GenericVariable(); + var.setType("int"); + assertEquals(ProductData.TYPE_INT32, var.getProductData()); + } + + @Test + public void setProductData_long() { + GenericVariable var = new GenericVariable(); + var.setType("long"); + assertEquals(ProductData.TYPE_INT64, var.getProductData()); + } + @Test + public void setProductData_float() { + GenericVariable var = new GenericVariable(); + var.setType("float"); + assertEquals(ProductData.TYPE_FLOAT32, var.getProductData()); + } + + @Test + public void setProductData_double() { + GenericVariable var = new GenericVariable(); + var.setType("double"); + assertEquals(ProductData.TYPE_FLOAT64, var.getProductData()); + } + + @Test + public void setProductData_string() { + GenericVariable var = new GenericVariable(); + var.setType("string"); + assertEquals(ProductData.TYPE_ASCII, var.getProductData()); + } + + @Test + public void setProductData_default() { + GenericVariable var = new GenericVariable(); + + try { + var.setType("non_existent_type"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Unsupported type: non_existent_type", e.getMessage()); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/StationDatabaseTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/StationDatabaseTest.java new file mode 100644 index 000000000..f20f3448d --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/generic/StationDatabaseTest.java @@ -0,0 +1,116 @@ +package com.bc.fiduceo.reader.insitu.generic; + +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + + +public class StationDatabaseTest { + + StationDatabase db; + + @Before + public void setUp() throws Exception { + GenericVariable var1 = new GenericVariable(); + GenericVariable var2 = new GenericVariable(); + GenericVariable var3 = new GenericVariable(); + var1.setName("id"); + var1.setType("string"); + var2.setName("latitude"); + var2.setType("long"); + var3.setName("station"); + var3.setType("string"); + + List station1 = new ArrayList<>(); + List station2 = new ArrayList<>(); + List station3 = new ArrayList<>(); + station1.add("id1"); + station1.add((long) 23.0); + station1.add("stat1"); + station2.add("id2"); + station2.add(-48.4); + station2.add("stat2"); + station3.add("id1"); + station3.add((long) -54.4); + station3.add("stat3"); + + db = new StationDatabase(); + db.setPrimaryId("id"); + db.setVariables(Arrays.asList(var1, var2, var3)); + db.setStations(Arrays.asList(station1, station2, station3)); + } + + @Test + public void extractRecord_noSecondaryId_success() { + GenericRecord record = db.extractRecord("id1", null); + + assertNotNull(record); + assertEquals(3, record.getValues().size()); + Object recordId = record.getValues().get("id"); + Object recordLatitude = record.getValues().get("latitude"); + Object recordStation = record.getValues().get("station"); + assertNotNull(recordId); + assertNotNull(recordLatitude); + assertNotNull(recordStation); + assertEquals("id1", recordId); + assertEquals((long) 23.0, recordLatitude); + assertEquals("stat1", recordStation); + } + + @Test + public void extractRecord_withSecondaryId_success() { + db.setSecondaryId("station"); + GenericRecord record = db.extractRecord("id1", "stat3"); + + assertNotNull(record); + assertEquals(3, record.getValues().size()); + Object recordId = record.getValues().get("id"); + Object recordLatitude = record.getValues().get("latitude"); + Object recordStation = record.getValues().get("station"); + assertNotNull(recordId); + assertNotNull(recordLatitude); + assertNotNull(recordStation); + assertEquals("id1", recordId); + assertEquals((long) -54.4, recordLatitude); + assertEquals("stat3", recordStation); + } + + + @Test + public void extractRecord_error_doesNotContainVariable() { + db.setPrimaryId("invalidIdentifier"); + try { + db.extractRecord("id2", null); + fail(); + } catch (IllegalStateException e) { + assertEquals("Station database does not contain variable 'invalidIdentifier'.", e.getMessage()); + } + } + + @Test + public void extractRecord_error_doesNotContainSecondaryVariable() { + db.setPrimaryId("id"); + db.setSecondaryId("invalidIdentifier"); + try { + db.extractRecord("id1", "station"); + fail(); + } catch (IllegalStateException e) { + assertEquals("Station database does not contain variable 'invalidIdentifier'.", e.getMessage()); + } + } + + @Test + public void extractRecord_error_doesNotStationForId() { + try { + db.extractRecord("id3", "stat4"); + fail(); + } catch (IllegalStateException e) { + assertEquals("Station database does not contain site/station with ids 'id'/'null.", e.getMessage()); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcCWReaderPluginTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcCWReaderPluginTest.java index 84994ee25..24671bcd6 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcCWReaderPluginTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcCWReaderPluginTest.java @@ -2,6 +2,7 @@ import com.bc.fiduceo.reader.DataType; import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.insitu.generic.GenericCsvReader; import org.junit.Before; import org.junit.Test; @@ -34,6 +35,6 @@ public void testGetDataType() { public void testCreateReader() { final Reader reader = plugin.createReader(null); assertNotNull(reader); - assertTrue(reader instanceof NdbcCWReader); + assertTrue(reader instanceof GenericCsvReader); } } diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderPluginTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderPluginTest.java index 2ff37707a..9e5ba13c5 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderPluginTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderPluginTest.java @@ -2,6 +2,7 @@ import com.bc.fiduceo.reader.DataType; import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.insitu.generic.GenericCsvReader; import org.junit.Before; import org.junit.Test; @@ -34,6 +35,6 @@ public void testGetDataType() { public void testCreateReader() { final Reader reader = plugin.createReader(null); assertNotNull(reader); - assertTrue(reader instanceof NdbcSMReader); + assertTrue(reader instanceof GenericCsvReader); } } diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderTest.java index 637b8c82a..8a2363927 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReaderTest.java @@ -94,7 +94,7 @@ record = reader.parseLine(line, calendar); public void testGetVariables() throws InvalidRangeException, IOException { final List variables = reader.getVariables(); - //assertEquals(0, variables.size()); + assertEquals(23, variables.size()); Variable variable = variables.get(0); assertEquals("station_id", variable.getShortName()); assertEquals(DataType.STRING, variable.getDataType()); @@ -121,21 +121,26 @@ public void testGetVariables() throws InvalidRangeException, IOException { assertEquals("wind_from_direction", variable.findAttribute(CF_STANDARD_NAME).getStringValue()); variable = variables.get(13); + assertEquals("WTMP", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + assertEquals("Sea surface temperature (Celsius). For buoys the depth is referenced to the hull's waterline. For fixed platforms it varies with tide, but is referenced to, or near Mean Lower Low Water (MLLW).", variable.findAttribute(CF_LONG_NAME).getStringValue()); + + variable = variables.get(14); assertEquals("WVHT", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); assertEquals("Significant wave height (meters) is calculated as the average of the highest one-third of all of the wave heights during the 20-minute sampling period.", variable.findAttribute(CF_LONG_NAME).getStringValue()); - variable = variables.get(16); + variable = variables.get(17); assertEquals("MWD", variable.getShortName()); assertEquals(DataType.SHORT, variable.getDataType()); assertEquals("degT", variable.findAttribute(CF_UNITS_NAME).getStringValue()); - variable = variables.get(19); + variable = variables.get(20); assertEquals("DEWP", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); assertEquals(999.f, variable.findAttribute(CF_FILL_VALUE_NAME).getNumericValue().floatValue(), 1e-8); - variable = variables.get(21); + variable = variables.get(22); assertEquals("TIDE", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); assertEquals("The water level in feet above or below Mean Lower Low Water (MLLW).", variable.findAttribute(CF_LONG_NAME).getStringValue()); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReader_IO_Test.java index 0064dbdb1..b84bb001f 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReader_IO_Test.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/ndbc/NdbcSMReader_IO_Test.java @@ -226,6 +226,10 @@ public void testReadRaw_1x1_coastBuoy() throws InvalidRangeException, IOExceptio assertEquals(DataType.SHORT, array.getDataType()); assertEquals(38, array.getShort(0)); + array = reader.readRaw(8, 2013, new Interval(1, 1), "WTMP"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(999.f, array.getFloat(0), 1e-8); + array = reader.readRaw(8, 2014, new Interval(1, 1), "WVHT"); assertEquals(DataType.FLOAT, array.getDataType()); assertEquals(99.f, array.getFloat(0), 1e-8); @@ -264,6 +268,10 @@ public void testReadScaled_1x1_oceanStation() throws InvalidRangeException, IOEx array = reader.readRaw(15, 2020, new Interval(1, 1), "WSPD"); assertEquals(DataType.FLOAT, array.getDataType()); assertEquals(5.2f, array.getFloat(0), 1e-8); + + array = reader.readRaw(15, 2020, new Interval(1, 1), "WTMP"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(999.f, array.getFloat(0), 1e-8); } finally { reader.close(); } diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/AMSR2SectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/AMSR2SectionParserTest.java index 1c907f4e8..b3c875b96 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/AMSR2SectionParserTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/AMSR2SectionParserTest.java @@ -27,7 +27,7 @@ public void testGetVariables() { assertEquals(variables.size(), parser.getNumVariables()); Variable variable = variables.get(0); - assertEquals("AMSR2_longitude", variable.getShortName()); + assertEquals("AMSR2_latitude", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(2); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParserTest.java index 5d2aaffd0..cd2484552 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParserTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParserTest.java @@ -1,6 +1,7 @@ package com.bc.fiduceo.reader.insitu.sic_cci; import com.bc.fiduceo.TestUtil; +import org.esa.snap.core.util.StringUtils; import org.junit.Before; import org.junit.Test; import ucar.ma2.Array; @@ -33,11 +34,11 @@ public void testParseTime() throws ParseException { @Test public void testParse() throws ParseException { - final String[] tokens = {"-66.33530", "-4.32590", "2015-12-15T20:00:00Z", "ASPeCt_ANT-XXXI_2_Polarstern", + String[] tokens = {"-66.33530", "-4.32590", "2015-12-15T20:00:00Z", "ASPeCt_ANT-XXXI_2_Polarstern", "30", "10", "70", "0.9", "0.55", "1.0", "6", "0.20", "10", "70", "0.8", "0.25", "1.0", "6", "0.20", "10", "70", "0.70", "0.15", "1.0", "6", "0.1", "-1.49", "-2.4", "2.7", "267", "97", "8", "2"}; - final Section section = parser.parse(tokens, 0); + Section section = parser.parse(tokens, 0); assertEquals(-66.33530f, section.get("latitude").getFloat(0), 1e-8); assertEquals(-4.32590f, section.get("longitude").getFloat(0), 1e-8); assertEquals(1450209600, section.get("time").getInt(0)); @@ -85,7 +86,7 @@ public void testGetVariables() { assertEquals(variables.size(), parser.getNumVariables()); Variable variable = variables.get(0); - assertEquals("longitude", variable.getShortName()); + assertEquals("latitude", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(3); @@ -94,11 +95,11 @@ public void testGetVariables() { variable = variables.get(4); assertEquals("SIC-total", variable.getShortName()); - assertEquals(DataType.BYTE, variable.getDataType()); + assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(5); assertEquals("SIC-primary", variable.getShortName()); - assertEquals(DataType.BYTE, variable.getDataType()); + assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(6); assertEquals("Ice-type-primary", variable.getShortName()); @@ -126,7 +127,7 @@ public void testGetVariables() { variable = variables.get(12); assertEquals("SIC-secondary", variable.getShortName()); - assertEquals(DataType.BYTE, variable.getDataType()); + assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(13); assertEquals("Ice-type-secondary", variable.getShortName()); @@ -154,7 +155,7 @@ public void testGetVariables() { variable = variables.get(19); assertEquals("SIC-tertiary", variable.getShortName()); - assertEquals(DataType.BYTE, variable.getDataType()); + assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(20); assertEquals("Ice-type-tertiary", variable.getShortName()); @@ -194,7 +195,7 @@ public void testGetVariables() { variable = variables.get(29); assertEquals("Wind-direction", variable.getShortName()); - assertEquals(DataType.SHORT, variable.getDataType()); + assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(30); assertEquals("Visibility", variable.getShortName()); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ASCATSectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ASCATSectionParserTest.java index 24a0d03cf..ea3e38c88 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ASCATSectionParserTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ASCATSectionParserTest.java @@ -28,7 +28,7 @@ public void testGetVariables() { assertEquals(variables.size(), parser.getNumVariables()); Variable variable = variables.get(0); - assertEquals("ASCAT_longitude", variable.getShortName()); + assertEquals("ASCAT_latitude", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(4); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SectionParserTest.java index ab32c93e5..6633c9ec1 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SectionParserTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SectionParserTest.java @@ -55,11 +55,11 @@ public void testGetVariables() { assertEquals(variables.size(), parser.getNumVariables()); Variable variable = variables.get(0); - assertEquals("longitude", variable.getShortName()); + assertEquals("latitude", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(1); - assertEquals("latitude", variable.getShortName()); + assertEquals("longitude", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(2); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SectionParserTest.java index 79bc388b3..3fd50dd88 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SectionParserTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SectionParserTest.java @@ -56,11 +56,11 @@ public void testGetVariables() { assertEquals(variables.size(), parser.getNumVariables()); Variable variable = variables.get(0); - assertEquals("longitude", variable.getShortName()); + assertEquals("latitude", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(1); - assertEquals("latitude", variable.getShortName()); + assertEquals("longitude", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(2); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ERASectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ERASectionParserTest.java index 628a2aea4..4ccd07e93 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ERASectionParserTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ERASectionParserTest.java @@ -22,7 +22,7 @@ public void testGetVariables_ERA() { assertEquals(variables.size(), parser.getNumVariables()); Variable variable = variables.get(0); - assertEquals("ERA_longitude", variable.getShortName()); + assertEquals("ERA_latitude", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(4); @@ -83,7 +83,7 @@ public void testGetVariables_ERA5() { assertEquals(variables.size(), parser.getNumVariables()); Variable variable = variables.get(1); - assertEquals("ERA5_latitude", variable.getShortName()); + assertEquals("ERA5_longitude", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(5); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/QSCATSectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/QSCATSectionParserTest.java index 93ecd7fda..bfc6a4f54 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/QSCATSectionParserTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/QSCATSectionParserTest.java @@ -28,7 +28,7 @@ public void testGetVariables() { assertEquals(variables.size(), parser.getNumVariables()); Variable variable = variables.get(0); - assertEquals("QSCAT_longitude", variable.getShortName()); + assertEquals("QSCAT_latitude", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(4); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SMOSSectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SMOSSectionParserTest.java index 1a5d0f661..22dbdd8a1 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SMOSSectionParserTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SMOSSectionParserTest.java @@ -28,7 +28,7 @@ public void testGetVariables() { assertEquals(variables.size(), parser.getNumVariables()); Variable variable = variables.get(1); - assertEquals("SMOS_latitude", variable.getShortName()); + assertEquals("SMOS_longitude", variable.getShortName()); assertEquals(DataType.FLOAT, variable.getDataType()); variable = variables.get(4); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SciCciInsituReader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SciCciInsituReader_IO_Test.java index 21703bf8b..7be5a80b4 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SciCciInsituReader_IO_Test.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SciCciInsituReader_IO_Test.java @@ -84,7 +84,7 @@ public void testGetVariables_DTUSIC1() throws IOException, InvalidRangeException assertEquals(66, variables.size()); Variable variable = variables.get(0); - assertEquals("longitude", variable.getShortName()); + assertEquals("latitude", variable.getShortName()); NCTestUtils.assertAttribute(variable, CF_FILL_VALUE_NAME, "9.969209968386869E36"); variable = variables.get(5); @@ -129,8 +129,8 @@ public void testGetVariables_DMISIC0() throws IOException, InvalidRangeException assertEquals(65, variables.size()); Variable variable = variables.get(0); - assertEquals("longitude", variable.getShortName()); - NCTestUtils.assertAttribute(variable, CF_UNITS_NAME, "degree_east"); + assertEquals("latitude", variable.getShortName()); + NCTestUtils.assertAttribute(variable, CF_UNITS_NAME, "degree_north"); variable = variables.get(4); assertEquals("SIC", variable.getShortName()); @@ -174,8 +174,8 @@ public void testGetVariables_ANTXXXI() throws IOException, InvalidRangeException assertEquals(93, variables.size()); Variable variable = variables.get(0); - assertEquals("longitude", variable.getShortName()); - NCTestUtils.assertAttribute(variable, CF_UNITS_NAME, "degree_east"); + assertEquals("latitude", variable.getShortName()); + NCTestUtils.assertAttribute(variable, CF_UNITS_NAME, "degree_north"); variable = variables.get(13); assertEquals("Ice-type-secondary", variable.getShortName()); @@ -240,7 +240,6 @@ public void testGetPixelLocator_DTUSIC1() throws IOException { try { reader.open(testFile); - reader.getPixelLocator(); fail("RuntimeException expected"); } catch (RuntimeException expected) { @@ -439,10 +438,10 @@ public void testReadRaw_3x3_ANTXXXI() throws InvalidRangeException, IOException // reference data Array array = reader.readRaw(7, 5, new Interval(3, 3), "SIC-total"); - assertEquals(DataType.BYTE, array.getDataType()); - NCTestUtils.assertValueAt(-127, 0, 0, array); + assertEquals(DataType.FLOAT, array.getDataType()); + NCTestUtils.assertValueAt(2147483647, 0, 0, array); NCTestUtils.assertValueAt(10, 1, 1, array); - NCTestUtils.assertValueAt(-127, 2, 2, array); + NCTestUtils.assertValueAt(2147483647, 2, 2, array); // ERA5 array = reader.readRaw(7, 6, new Interval(3, 3), "ERA_v10"); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsGtmba2InsituReaderPluginTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsGtmba2InsituReaderPluginTest.java new file mode 100644 index 000000000..43d0b59a3 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsGtmba2InsituReaderPluginTest.java @@ -0,0 +1,39 @@ +package com.bc.fiduceo.reader.insitu.sirds_sst; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class SirdsGtmba2InsituReaderPluginTest { + + private SirdsGtmba2InsituReaderPlugin plugin; + + @Before + public void setUp() { + plugin = new SirdsGtmba2InsituReaderPlugin(); + } + + @Test + public void testGetSupportedSensorKeys() throws Exception { + final String[] expected = {"gtmba2-sirds"}; + + final String[] sensorKeys = plugin.getSupportedSensorKeys(); + assertArrayEquals(expected, sensorKeys); + } + + @Test + public void testCreateReader() { + final Reader reader = plugin.createReader(null); + assertNotNull(reader); + assertTrue(reader instanceof SirdsInsituReader); + assertEquals("gtmba2-sirds", ((SirdsInsituReader) reader).getSensorKey()); + } + + @Test + public void testGetDataType() { + assertEquals(DataType.INSITU, plugin.getDataType()); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReaderTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReaderTest.java index be379612a..35f8b63a6 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReaderTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReaderTest.java @@ -51,6 +51,27 @@ public void testGetRegEx_argosurf() { assertFalse(matcher.matches()); } + @Test + public void testGetRegEx_gtmba2() { + final SirdsInsituReader insituReader = new SirdsInsituReader("gtmba2-sirds"); + final String expected = "SSTCCI2_refdata_gtmba2_\\d{6}.nc"; + + assertEquals(expected, insituReader.getRegEx()); + final Pattern pattern = java.util.regex.Pattern.compile(expected); + + Matcher matcher = pattern.matcher("SSTCCI2_refdata_gtmba2_200012.nc"); + assertTrue(matcher.matches()); + + matcher = pattern.matcher("SSTCCI2_refdata_gtmba_200801.nc"); + assertFalse(matcher.matches()); + + matcher = pattern.matcher("insitu_5_WMOID_5901880_20100514_20100627.nc"); + assertFalse(matcher.matches()); + + matcher = pattern.matcher("NSS.HIRX.TN.D79287.S1623.E1807.B0516566.GC.nc"); + assertFalse(matcher.matches()); + } + @Test public void testGetLongitudeVariableName() { final SirdsInsituReader insituReader = new SirdsInsituReader("whatever"); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReader_IO_Test.java index e1bac9441..7bebcd68b 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReader_IO_Test.java +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReader_IO_Test.java @@ -46,7 +46,7 @@ public void tearDown() throws Exception { @Test public void testReadAcquisitionInfo_drifter() throws Exception { insituReader = new SirdsInsituReader("drifter"); - openFile("SSTCCI2_refdata_drifter_201304.nc"); + openFile("SSTCCI2_refdata_drifter_201304.nc","v1.0"); final AcquisitionInfo info = insituReader.read(); assertNotNull(info); @@ -62,7 +62,7 @@ public void testReadAcquisitionInfo_drifter() throws Exception { @Test public void testReadAcquisitionInfo_mooring() throws Exception { insituReader = new SirdsInsituReader("mooring"); - openFile("SSTCCI2_refdata_mooring_201602.nc"); + openFile("SSTCCI2_refdata_mooring_201602.nc","v1.0"); final AcquisitionInfo info = insituReader.read(); assertNotNull(info); @@ -75,10 +75,26 @@ public void testReadAcquisitionInfo_mooring() throws Exception { assertNull(info.getBoundingGeometry()); } + @Test + public void testReadAcquisitionInfo_gtmba2() throws Exception { + insituReader = new SirdsInsituReader("gtmba2-sirds"); + openFile("SSTCCI2_refdata_gtmba2_201204.nc", "raw"); + + final AcquisitionInfo info = insituReader.read(); + assertNotNull(info); + + TestUtil.assertCorrectUTCDate(2012, 4, 1, 0, 0, 0, 0, info.getSensingStart()); + TestUtil.assertCorrectUTCDate(2012, 4, 30, 23, 50, 0, 0, info.getSensingStop()); + + assertEquals(NodeType.UNDEFINED, info.getNodeType()); + + assertNull(info.getBoundingGeometry()); + } + @Test public void testReadAcquisitionInfo_xbt() throws Exception { insituReader = new SirdsInsituReader("xbt"); - openFile("SSTCCI2_refdata_xbt_200204.nc"); + openFile("SSTCCI2_refdata_xbt_200204.nc","v1.0"); final AcquisitionInfo info = insituReader.read(); assertNotNull(info); @@ -94,7 +110,7 @@ public void testReadAcquisitionInfo_xbt() throws Exception { @Test public void testGetVariables_drifter() throws Exception { insituReader = new SirdsInsituReader("drifter"); - openFile("SSTCCI2_refdata_drifter_201304.nc"); + openFile("SSTCCI2_refdata_drifter_201304.nc","v1.0"); final List variables = insituReader.getVariables(); assertNotNull(variables); @@ -111,10 +127,30 @@ public void testGetVariables_drifter() throws Exception { assertEquals("unique_id", variables.get(18).getShortName()); } + @Test + public void testGetVariables_gtmba2() throws Exception { + insituReader = new SirdsInsituReader("gtmba2-sirds"); + openFile("SSTCCI2_refdata_gtmba2_201204.nc","raw"); + final List variables = insituReader.getVariables(); + + assertNotNull(variables); + assertEquals(19, variables.size()); + assertEquals("subcol2", variables.get(2).getShortName()); + assertEquals("plat_id", variables.get(4).getShortName()); + assertEquals("latitude", variables.get(6).getShortName()); + assertEquals("depth_corr", variables.get(8).getShortName()); + assertEquals("sst_type_corr", variables.get(10).getShortName()); + assertEquals("sst_type_corr_unc", variables.get(11).getShortName()); + assertEquals("sst_rand_unc", variables.get(14).getShortName()); + assertEquals("qc1", variables.get(16).getShortName()); + assertEquals("qc2", variables.get(17).getShortName()); + assertEquals("unique_id", variables.get(18).getShortName()); + } + @Test public void testReadRaw_drifter() throws Exception { insituReader = new SirdsInsituReader("drifter"); - openFile("SSTCCI2_refdata_drifter_201304.nc"); + openFile("SSTCCI2_refdata_drifter_201304.nc","v1.0"); final Array array = insituReader.readRaw(0, 0, new Interval(1, 1), "depth"); assertNotNull(array); @@ -126,7 +162,7 @@ public void testReadRaw_drifter() throws Exception { @Test public void testReadRaw_mooring() throws Exception { insituReader = new SirdsInsituReader("mooring"); - openFile("SSTCCI2_refdata_mooring_201602.nc"); + openFile("SSTCCI2_refdata_mooring_201602.nc","v1.0"); final Array array = insituReader.readRaw(0, 1, new Interval(1, 1), "depth_corr"); assertNotNull(array); @@ -138,7 +174,7 @@ public void testReadRaw_mooring() throws Exception { @Test public void testReadRaw_xbt() throws Exception { insituReader = new SirdsInsituReader("xbt"); - openFile("SSTCCI2_refdata_xbt_200204.nc"); + openFile("SSTCCI2_refdata_xbt_200204.nc","v1.0"); final Array array = insituReader.readRaw(0, 2, new Interval(1, 1), "latitude"); assertNotNull(array); @@ -150,7 +186,7 @@ public void testReadRaw_xbt() throws Exception { @Test public void testReadRaw_drifter_3x3() throws Exception { insituReader = new SirdsInsituReader("drifter"); - openFile("SSTCCI2_refdata_drifter_201304.nc"); + openFile("SSTCCI2_refdata_drifter_201304.nc","v1.0"); final Array array = insituReader.readRaw(0, 3, new Interval(3, 3), "longitude"); assertNotNull(array); @@ -170,7 +206,7 @@ public void testReadRaw_drifter_3x3() throws Exception { @Test public void testReadRaw_drifter_3x3_plat_id() throws Exception { insituReader = new SirdsInsituReader("drifter"); - openFile("SSTCCI2_refdata_drifter_201304.nc"); + openFile("SSTCCI2_refdata_drifter_201304.nc","v1.0"); final Array array = insituReader.readRaw(0, 4, new Interval(3, 3), "plat_id"); final int[] shape = array.getShape(); @@ -183,7 +219,7 @@ public void testReadRaw_drifter_3x3_plat_id() throws Exception { @Test public void testReadRaw_mooring_3x3() throws Exception { insituReader = new SirdsInsituReader("mooring"); - openFile("SSTCCI2_refdata_mooring_201602.nc"); + openFile("SSTCCI2_refdata_mooring_201602.nc","v1.0"); final Array array = insituReader.readRaw(0, 4, new Interval(3, 3), "sst"); assertNotNull(array); @@ -203,7 +239,7 @@ public void testReadRaw_mooring_3x3() throws Exception { @Test public void testReadScaled_xbt() throws Exception { insituReader = new SirdsInsituReader("xbt"); - openFile("SSTCCI2_refdata_xbt_200204.nc"); + openFile("SSTCCI2_refdata_xbt_200204.nc","v1.0"); final Array array = insituReader.readScaled(0, 5, new Interval(1, 1), "sst_comb_unc"); assertNotNull(array); @@ -215,7 +251,7 @@ public void testReadScaled_xbt() throws Exception { @Test public void testReadScaled_drifter_3x3() throws Exception { insituReader = new SirdsInsituReader("drifter"); - openFile("SSTCCI2_refdata_drifter_201304.nc"); + openFile("SSTCCI2_refdata_drifter_201304.nc","v1.0"); final Array array = insituReader.readScaled(0, 6, new Interval(3, 3), "sst_rand_unc"); assertNotNull(array); @@ -235,7 +271,7 @@ public void testReadScaled_drifter_3x3() throws Exception { @Test public void testReadRaw_mooring_uniqueId() throws Exception { insituReader = new SirdsInsituReader("mooring"); - openFile("SSTCCI2_refdata_mooring_201602.nc"); + openFile("SSTCCI2_refdata_mooring_201602.nc","v1.0"); final Array idArray = insituReader.readRaw(0, 7, new Interval(1, 1), "unique_id"); assertNotNull(idArray); @@ -247,7 +283,7 @@ public void testReadRaw_mooring_uniqueId() throws Exception { @Test public void testReadRaw_xbt_uniqueId() throws Exception { insituReader = new SirdsInsituReader("xbt"); - openFile("SSTCCI2_refdata_xbt_200204.nc"); + openFile("SSTCCI2_refdata_xbt_200204.nc","v1.0"); final Array idArray = insituReader.readRaw(0, 8, new Interval(1, 1), "unique_id"); assertNotNull(idArray); @@ -259,7 +295,7 @@ public void testReadRaw_xbt_uniqueId() throws Exception { @Test public void testReadAcquisitionTime_drifter() throws Exception { insituReader = new SirdsInsituReader("drifter"); - openFile("SSTCCI2_refdata_drifter_201304.nc"); + openFile("SSTCCI2_refdata_drifter_201304.nc","v1.0"); final ArrayInt.D2 array = insituReader.readAcquisitionTime(0, 9, new Interval(3, 3)); assertNotNull(array); @@ -280,7 +316,7 @@ public void testReadAcquisitionTime_drifter() throws Exception { @Test public void testReadAcquisitionTime_mooring() throws Exception { insituReader = new SirdsInsituReader("mooring"); - openFile("SSTCCI2_refdata_mooring_201602.nc"); + openFile("SSTCCI2_refdata_mooring_201602.nc","v1.0"); final ArrayInt.D2 array = insituReader.readAcquisitionTime(0, 10, new Interval(1, 1)); assertNotNull(array); @@ -293,7 +329,7 @@ public void testReadAcquisitionTime_mooring() throws Exception { @Test public void testGetProductSize_xbt() throws Exception { insituReader = new SirdsInsituReader("xbt"); - openFile("SSTCCI2_refdata_xbt_200204.nc"); + openFile("SSTCCI2_refdata_xbt_200204.nc","v1.0"); final Dimension productSize = insituReader.getProductSize(); assertNotNull(productSize); @@ -305,7 +341,7 @@ public void testGetProductSize_xbt() throws Exception { @Test public void testGetProductSize_drifter() throws Exception { insituReader = new SirdsInsituReader("drifter"); - openFile("SSTCCI2_refdata_drifter_201304.nc"); + openFile("SSTCCI2_refdata_drifter_201304.nc","v1.0"); final Dimension productSize = insituReader.getProductSize(); assertNotNull(productSize); @@ -317,7 +353,7 @@ public void testGetProductSize_drifter() throws Exception { @Test public void testGetTimeLocator_mooring() throws Exception { insituReader = new SirdsInsituReader("mooring"); - openFile("SSTCCI2_refdata_mooring_201602.nc"); + openFile("SSTCCI2_refdata_mooring_201602.nc","v1.0"); final TimeLocator timeLocator = insituReader.getTimeLocator(); assertNotNull(timeLocator); @@ -329,7 +365,7 @@ public void testGetTimeLocator_mooring() throws Exception { @Test public void testGetTimeLocator_xbt() throws Exception { insituReader = new SirdsInsituReader("xbt"); - openFile("SSTCCI2_refdata_xbt_200204.nc"); + openFile("SSTCCI2_refdata_xbt_200204.nc", "v1.0"); final TimeLocator timeLocator = insituReader.getTimeLocator(); assertNotNull(timeLocator); @@ -341,7 +377,7 @@ public void testGetTimeLocator_xbt() throws Exception { @Test public void testGetPixelLocator() throws Exception { insituReader = new SirdsInsituReader("drifter"); - openFile("SSTCCI2_refdata_drifter_201304.nc"); + openFile("SSTCCI2_refdata_drifter_201304.nc", "v1.0"); try { insituReader.getPixelLocator(); @@ -353,7 +389,7 @@ public void testGetPixelLocator() throws Exception { @Test public void testGetSubScenePixelLocator() throws Exception { insituReader = new SirdsInsituReader("mooring"); - openFile("SSTCCI2_refdata_mooring_201602.nc"); + openFile("SSTCCI2_refdata_mooring_201602.nc","v1.0"); final GeometryFactory geometryFactory = new GeometryFactory(GeometryFactory.Type.S2); final Polygon polygon = geometryFactory.createPolygon(Arrays.asList( geometryFactory.createPoint(4, 5), @@ -368,8 +404,8 @@ public void testGetSubScenePixelLocator() throws Exception { } } - private void openFile(String fileName) throws IOException { - final String testFilePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "sirds", "v1.0", fileName}, false); + private void openFile(String fileName, String version) throws IOException { + final String testFilePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "sirds", version, fileName}, false); final File insituDataFile = TestUtil.getTestDataFileAsserted(testFilePath); insituReader.open(insituDataFile); diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/TaoReaderPluginTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/TaoReaderPluginTest.java new file mode 100644 index 000000000..a79253990 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/TaoReaderPluginTest.java @@ -0,0 +1,36 @@ +package com.bc.fiduceo.reader.insitu.tao; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class TaoReaderPluginTest { + + private TaoReaderPlugin plugin; + + @Before + public void setUp() { + plugin = new TaoReaderPlugin(); + } + + @Test + public void testGetSupportedKeys() { + final String[] expected = {"tao-sss", "pirata-sss", "rama-sss"}; + + assertArrayEquals(expected, plugin.getSupportedSensorKeys()); + } + + @Test + public void testGetDataType() { + assertEquals(DataType.INSITU, plugin.getDataType()); + } + + @Test + public void testCreateReader(){ + final Reader reader = plugin.createReader(null); + assertTrue(reader instanceof TaoReader); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/TaoReaderTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/TaoReaderTest.java new file mode 100644 index 000000000..49018a296 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/TaoReaderTest.java @@ -0,0 +1,251 @@ +package com.bc.fiduceo.reader.insitu.tao; + +import com.bc.fiduceo.geometry.GeometryFactory; +import com.bc.fiduceo.geometry.Polygon; +import org.junit.Before; +import org.junit.Test; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.bc.fiduceo.util.NetCDFUtils.*; +import static org.junit.Assert.*; + +public class TaoReaderTest { + + private TaoReader reader; + + @Before + public void setUp() { + reader = new TaoReader(); + } + + @Test + public void testGetRegEx() { + final String expected = "(?:TAO|TRITON|RAMA|PIRATA)_\\w+_\\w+(-\\w+)??\\d{4}-\\d{2}.txt"; + + assertEquals(expected, reader.getRegEx()); + final Pattern pattern = java.util.regex.Pattern.compile(expected); + + Matcher matcher = pattern.matcher("TRITON_TR0N156E_1998_2016-07.txt"); + assertTrue(matcher.matches()); + + matcher = pattern.matcher("TAO_T0N170W_DM207A-20160829_2017-04.txt"); + assertTrue(matcher.matches()); + + matcher = pattern.matcher("TAO_T2S110W_DM233A-20170608_2018-02.txt"); + assertTrue(matcher.matches()); + + matcher = pattern.matcher("PIRATA_0N35W_sss_2017-08.txt"); + assertTrue(matcher.matches()); + + matcher = pattern.matcher("RAMA_4S805E_sss_2017-10.txt"); + assertTrue(matcher.matches()); + + matcher = pattern.matcher("PIRATA_0N35W_sss_2016-10.txt"); + assertTrue(matcher.matches()); + } + + @Test + public void testGetLongitudeVariableName() { + assertEquals("longitude", reader.getLongitudeVariableName()); + } + + @Test + public void testGetLatitudeVariableName() { + assertEquals("latitude", reader.getLatitudeVariableName()); + } + + @Test + public void testGetVariables() throws InvalidRangeException, IOException { + final List variables = reader.getVariables(); + + assertEquals(13, variables.size()); + + // --- longitude --- + Variable variable = variables.get(0); + assertEquals("longitude", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + + Attribute attribute = variable.findAttribute(CF_STANDARD_NAME); + assertNotNull(attribute); + assertEquals("longitude", attribute.getStringValue()); + + // --- latitude --- + variable = variables.get(1); + assertEquals("latitude", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + + attribute = variable.findAttribute(CF_UNITS_NAME); + assertNotNull(attribute); + assertEquals("degree_north", attribute.getStringValue()); + + // --- time --- + variable = variables.get(2); + assertEquals("time", variable.getShortName()); + assertEquals(DataType.INT, variable.getDataType()); + + attribute = variable.findAttribute(CF_STANDARD_NAME); + assertNotNull(attribute); + assertEquals("time", attribute.getStringValue()); + + // --- SSS --- + variable = variables.get(3); + assertEquals("SSS", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + + attribute = variable.findAttribute(CF_FILL_VALUE_NAME); + assertNotNull(attribute); + assertEquals(-9.999f, attribute.getNumericValue().floatValue(), 1e-8); + + // --- SST --- + variable = variables.get(4); + assertEquals("SST", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + + attribute = variable.findAttribute(CF_UNITS_NAME); + assertNotNull(attribute); + assertEquals("degree_Celsius", attribute.getStringValue()); + + // --- AIRT --- + variable = variables.get(5); + assertEquals("AIRT", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + + attribute = variable.findAttribute(CF_STANDARD_NAME); + assertNotNull(attribute); + assertEquals("air_temperature", attribute.getStringValue()); + + // --- RH --- + variable = variables.get(6); + assertEquals("RH", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + + attribute = variable.findAttribute(CF_FILL_VALUE_NAME); + assertNotNull(attribute); + assertEquals(-9.99f, attribute.getNumericValue().floatValue(), 1e-8); + + // --- WSPD --- + variable = variables.get(7); + assertEquals("WSPD", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + + attribute = variable.findAttribute(CF_UNITS_NAME); + assertNotNull(attribute); + assertEquals("m/s", attribute.getStringValue()); + + // --- WDIR --- + variable = variables.get(8); + assertEquals("WDIR", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + + attribute = variable.findAttribute(CF_STANDARD_NAME); + assertNotNull(attribute); + assertEquals("wind_to_direction", attribute.getStringValue()); + + // --- BARO --- + variable = variables.get(9); + assertEquals("BARO", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + + attribute = variable.findAttribute(CF_FILL_VALUE_NAME); + assertNotNull(attribute); + assertEquals(-9.9f, attribute.getNumericValue().floatValue(), 1e-8); + + // --- RAIN --- + variable = variables.get(10); + assertEquals("RAIN", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + + attribute = variable.findAttribute(CF_UNITS_NAME); + assertNotNull(attribute); + assertEquals("mm/hour", attribute.getStringValue()); + + // --- Q --- + variable = variables.get(11); + assertEquals("Q", variable.getShortName()); + assertEquals(DataType.INT, variable.getDataType()); + + attribute = variable.findAttribute(CF_LONG_NAME); + assertNotNull(attribute); + assertEquals("Data Quality Codes", attribute.getStringValue()); + + // --- M --- + variable = variables.get(12); + assertEquals("M", variable.getShortName()); + assertEquals(DataType.STRING, variable.getDataType()); + + attribute = variable.findAttribute(CF_LONG_NAME); + assertNotNull(attribute); + assertEquals("Data Mode Codes", attribute.getStringValue()); + } + + @Test + public void testParseLine() { + final TaoRecord record = TaoReader.parseLine("1464854400 -139.99 -2.0402594 35.461 -9.999 25.63 79.46 5.4 273.0 -9.9 -9.99 19111199 DDDDDDDD"); + assertNotNull(record); + + assertEquals(1464854400, record.time); + assertEquals(-139.99f, record.longitude, 1e-8); + assertEquals(-2.0402594f, record.latitude, 1e-8); + assertEquals(35.461f, record.SSS, 1e-8); + assertEquals(-9.999f, record.SST, 1e-8); + assertEquals(25.63f, record.AIRT, 1e-8); + assertEquals(79.46f, record.RH, 1e-8); + assertEquals(5.4f, record.WSPD, 1e-8); + assertEquals(273.f, record.WDIR, 1e-8); + assertEquals(-9.9f, record.BARO, 1e-8); + assertEquals(-9.99f, record.RAIN, 1e-8); + assertEquals(19111199, record.Q); + assertEquals("DDDDDDDD", record.M); + } + + @Test + public void testGetPixelLocator() throws IOException { + try { + reader.getPixelLocator(); + fail("RuntimeException expected"); + } catch (RuntimeException expected) { + // expected + } + } + + @Test + public void testGetSubScenePixelLocator() throws IOException { + final GeometryFactory geometryFactory = new GeometryFactory(GeometryFactory.Type.S2); + final Polygon polygon = geometryFactory.createPolygon(Arrays.asList( + geometryFactory.createPoint(6, 9), + geometryFactory.createPoint(7, 0), + geometryFactory.createPoint(7, 10) + )); + + try { + reader.getSubScenePixelLocator(polygon); + fail("RuntimeException expected"); + } catch (RuntimeException expected) { + // expected + } + } + + @Test + public void testExtractYMDFromFileName() { + int[] ymd = reader.extractYearMonthDayFromFilename("TRITON_TR0N156E_1998_2017-10.txt"); + assertEquals(3, ymd.length); + assertEquals(2017, ymd[0]); + assertEquals(10, ymd[1]); + assertEquals(1, ymd[2]); + + ymd = reader.extractYearMonthDayFromFilename("RAMA_12N90E_sss_2017-09.txt"); + assertEquals(3, ymd.length); + assertEquals(2017, ymd[0]); + assertEquals(9, ymd[1]); + assertEquals(1, ymd[2]); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/TaoReader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/TaoReader_IO_Test.java new file mode 100644 index 000000000..0707bb8f0 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/TaoReader_IO_Test.java @@ -0,0 +1,472 @@ +package com.bc.fiduceo.reader.insitu.tao; + +import com.bc.fiduceo.IOTestRunner; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.core.NodeType; +import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.time.TimeLocator; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.Array; +import ucar.ma2.ArrayInt; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.*; + +@RunWith(IOTestRunner.class) +public class TaoReader_IO_Test { + + private TaoReader reader; + + @Before + public void setUp() { + reader = new TaoReader(); + } + + @After + public void tearDown() throws IOException { + reader.close(); + } + + @Test + public void testReadAcquisitionInfo_TAO() throws Exception { + final File insituDataFile = getTAOProduct(); + + reader.open(insituDataFile); + + final AcquisitionInfo info = reader.read(); + TestUtil.assertCorrectUTCDate(2016, 6, 1, 0, 0, 0, 0, info.getSensingStart()); + TestUtil.assertCorrectUTCDate(2016, 6, 30, 23, 0, 0, 0, info.getSensingStop()); + + assertEquals(NodeType.UNDEFINED, info.getNodeType()); + + assertNull(info.getBoundingGeometry()); + } + + + @Test + public void testReadAcquisitionInfo_TRITON() throws Exception { + final File insituDataFile = getTritonProduct(); + + reader.open(insituDataFile); + + final AcquisitionInfo info = reader.read(); + TestUtil.assertCorrectUTCDate(2017, 10, 1, 12, 0, 0, 0, info.getSensingStart()); + TestUtil.assertCorrectUTCDate(2017, 10, 31, 12, 0, 0, 0, info.getSensingStop()); + + assertEquals(NodeType.UNDEFINED, info.getNodeType()); + + assertNull(info.getBoundingGeometry()); + } + + @Test + public void testReadAcquisitionInfo_PIRATA() throws Exception { + final File insituDataFile = getPirataProduct(); + + reader.open(insituDataFile); + + final AcquisitionInfo info = reader.read(); + TestUtil.assertCorrectUTCDate(2016, 10, 1, 0, 0, 0, 0, info.getSensingStart()); + TestUtil.assertCorrectUTCDate(2016, 10, 31, 23, 0, 0, 0, info.getSensingStop()); + + assertEquals(NodeType.UNDEFINED, info.getNodeType()); + + assertNull(info.getBoundingGeometry()); + } + + @Test + public void testGetProductSize_TAO() throws IOException { + final File insituDataFile = getTAOProduct(); + + reader.open(insituDataFile); + + final Dimension productSize = reader.getProductSize(); + + assertNotNull(productSize); + assertEquals("product_size", productSize.getName()); + assertEquals(1, productSize.getNx()); + assertEquals(720, productSize.getNy()); + } + + @Test + public void testGetProductSize_TRITON() throws IOException { + final File insituDataFile = getTritonProduct(); + + reader.open(insituDataFile); + + final Dimension productSize = reader.getProductSize(); + + assertNotNull(productSize); + assertEquals("product_size", productSize.getName()); + assertEquals(1, productSize.getNx()); + assertEquals(31, productSize.getNy()); + } + + @Test + public void testGetProductSize_RAMA() throws IOException { + final File insituDataFile = getRamaProduct(); + + reader.open(insituDataFile); + + final Dimension productSize = reader.getProductSize(); + + assertNotNull(productSize); + assertEquals("product_size", productSize.getName()); + assertEquals(1, productSize.getNx()); + assertEquals(720, productSize.getNy()); + } + + @Test + public void testGetTimeLocator_TAO() throws IOException { + final File insituDataFile = getTAOProduct(); + + reader.open(insituDataFile); + + final TimeLocator timeLocator = reader.getTimeLocator(); + assertEquals(1464739200000L, timeLocator.getTimeFor(0, 0)); + assertEquals(1465664400000L, timeLocator.getTimeFor(10, 257)); + assertEquals(1466982000000L, timeLocator.getTimeFor(20, 623)); + + assertEquals(-1L, timeLocator.getTimeFor(20, -1)); + assertEquals(-1L, timeLocator.getTimeFor(20, 2567)); + } + + @Test + public void testGetTimeLocator_TRITON() throws IOException { + final File insituDataFile = getTritonProduct(); + + reader.open(insituDataFile); + + final TimeLocator timeLocator = reader.getTimeLocator(); + assertEquals(1506859200000L, timeLocator.getTimeFor(0, 0)); + assertEquals(1508155200000L, timeLocator.getTimeFor(10, 15)); + assertEquals(1509451200000L, timeLocator.getTimeFor(20, 30)); + + assertEquals(-1L, timeLocator.getTimeFor(20, -1)); + assertEquals(-1L, timeLocator.getTimeFor(20, 45)); + } + + @Test + public void testGetTimeLocator_PIRATA() throws IOException { + final File insituDataFile = getPirataProduct(); + + reader.open(insituDataFile); + + final TimeLocator timeLocator = reader.getTimeLocator(); + assertEquals(1475280000000L, timeLocator.getTimeFor(0, 0)); + assertEquals(1475337600000L, timeLocator.getTimeFor(10, 16)); + assertEquals(1475391600000L, timeLocator.getTimeFor(20, 31)); + + assertEquals(-1L, timeLocator.getTimeFor(20, -1)); + assertEquals(-1L, timeLocator.getTimeFor(20, 1734)); + } + + @Test + public void testReadRaw_TAO() throws IOException, InvalidRangeException { + final File insituDataFile = getTAOProduct(); + + reader.open(insituDataFile); + + final Interval singlePx = new Interval(1, 1); + + // x is ignored + Array array = reader.readRaw(8, 451, singlePx, "time"); + assertEquals(DataType.INT, array.getDataType()); + assertEquals(1466362800, array.getInt(0)); + + array = reader.readRaw(9, 452, singlePx, "longitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-140.02f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 453, singlePx, "latitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-2.04f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 454, singlePx, "SSS"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(35.631f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 455, singlePx, "SST"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(27.316f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 456, singlePx, "AIRT"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(26.83f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 457, singlePx, "RH"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(82.99f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 458, singlePx, "WSPD"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(8.1f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 459, singlePx, "WDIR"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(265.f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 460, singlePx, "BARO"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-9.9f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 461, singlePx, "RAIN"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-9.99f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 462, singlePx, "Q"); + assertEquals(DataType.INT, array.getDataType()); + assertEquals(11111199, array.getInt(0)); + + array = reader.readRaw(9, 463, singlePx, "M"); + assertEquals(DataType.STRING, array.getDataType()); + assertEquals("DDDDDDDD", array.getObject(0)); + } + + @Test + public void testReadRaw_TRITON_3x3() throws IOException, InvalidRangeException { + final File insituDataFile = getTritonProduct(); + + reader.open(insituDataFile); + + Array array = reader.readRaw(9, 19, new Interval(3, 3), "SSS"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-9.999f, array.getFloat(0), 1e-8); + assertEquals(35.113f, array.getFloat(4), 1e-8); + assertEquals(-9.999f, array.getFloat(8), 1e-8); + } + + @Test + public void testReadRaw_RAMA() throws IOException, InvalidRangeException { + final File insituDataFile = getRamaProduct(); + + reader.open(insituDataFile); + + final Interval singlePx = new Interval(1, 1); + + // x is ignored + Array array = reader.readRaw(8, 452, singlePx, "time"); + assertEquals(DataType.INT, array.getDataType()); + assertEquals(1524168000, array.getInt(0)); + + array = reader.readRaw(9, 453, singlePx, "longitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(80.5f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 454, singlePx, "latitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-8.f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 455, singlePx, "SSS"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(34.251f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 456, singlePx, "SST"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(28.6f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 457, singlePx, "AIRT"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(27.6f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 458, singlePx, "RH"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(84.1f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 459, singlePx, "WSPD"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(6.6f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 460, singlePx, "WDIR"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(295.4f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 461, singlePx, "BARO"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(1010.6f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 462, singlePx, "RAIN"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(0.f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 463, singlePx, "Q"); + assertEquals(DataType.INT, array.getDataType()); + assertEquals(22222222, array.getInt(0)); + + array = reader.readRaw(9, 464, singlePx, "M"); + assertEquals(DataType.STRING, array.getDataType()); + assertEquals("11111111", array.getObject(0)); + } + + @Test + public void testReadScaled_TRITON() throws IOException, InvalidRangeException { + final File insituDataFile = getTritonProduct(); + + reader.open(insituDataFile); + + final Interval singlePx = new Interval(1, 1); + + // x is ignored + Array array = reader.readScaled(8, 5, singlePx, "time"); + assertEquals(DataType.INT, array.getDataType()); + assertEquals(1507291200, array.getInt(0)); + + array = reader.readScaled(9, 6, singlePx, "longitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(156.f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 7, singlePx, "latitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(0.f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 8, singlePx, "SSS"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(34.877f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 9, singlePx, "SST"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(29.69f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 10, singlePx, "AIRT"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(28.79f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 11, singlePx, "RH"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(78.37f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 12, singlePx, "WSPD"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(7.f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 13, singlePx, "WDIR"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(300.7f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 14, singlePx, "BARO"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(1005.3f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 15, singlePx, "RAIN"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(0.f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 16, singlePx, "Q"); + assertEquals(DataType.INT, array.getDataType()); + assertEquals(22222222, array.getInt(0)); + + array = reader.readScaled(9, 17, singlePx, "M"); + assertEquals(DataType.STRING, array.getDataType()); + assertEquals("MMMMMMMM", array.getObject(0)); + } + + @Test + public void testReadScaled_PIRATA() throws IOException, InvalidRangeException { + final File insituDataFile = getPirataProduct(); + + reader.open(insituDataFile); + + final Interval singlePx = new Interval(1, 1); + + // x is ignored + Array array = reader.readScaled(8, 6, singlePx, "time"); + assertEquals(DataType.INT, array.getDataType()); + assertEquals(1475301600, array.getInt(0)); + + array = reader.readScaled(9, 7, singlePx, "longitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-35.f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 8, singlePx, "latitude"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(0.f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 9, singlePx, "SSS"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(35.649f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 10, singlePx, "SST"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-9.999f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 11, singlePx, "AIRT"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-9.99f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 12, singlePx, "RH"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-9.99f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 13, singlePx, "WSPD"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-99.9f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 14, singlePx, "WDIR"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-99.9f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 15, singlePx, "BARO"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-9.9f, array.getFloat(0), 1e-8); + + array = reader.readScaled(9, 16, singlePx, "RAIN"); + assertEquals(DataType.FLOAT, array.getDataType()); + assertEquals(-9.99f, array.getFloat(0), 1e-8); + + array = reader.readRaw(9, 17, singlePx, "Q"); + assertEquals(DataType.INT, array.getDataType()); + assertEquals(39999999, array.getInt(0)); + + array = reader.readScaled(9, 18, singlePx, "M"); + assertEquals(DataType.STRING, array.getDataType()); + assertEquals("5DDDDDDD", array.getObject(0)); + } + + @Test + public void testReadAcquisitionTime_TAO() throws IOException, InvalidRangeException { + final File insituDataFile = getTAOProduct(); + + reader.open(insituDataFile); + + ArrayInt.D2 acquisitionTime = reader.readAcquisitionTime(3, 701, new Interval(1, 1)); + assertEquals(1467262800, acquisitionTime.getInt(0)); + } + + @Test + public void testReadAcquisitionTime_RAMA() throws IOException, InvalidRangeException { + final File insituDataFile = getRamaProduct(); + + reader.open(insituDataFile); + + ArrayInt.D2 acquisitionTime = reader.readAcquisitionTime(3, 702, new Interval(1, 1)); + assertEquals(1525068000, acquisitionTime.getInt(0)); + } + + private static File getTAOProduct() throws IOException { + final String testFilePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "tao-sss", "v1", "2016", "06", "TAO_T2S140W_DM167A-20160228_2016-06.txt"}, false); + return TestUtil.getTestDataFileAsserted(testFilePath); + } + + private static File getTritonProduct() throws IOException { + final String testFilePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "tao-sss", "v1", "2017", "10", "TRITON_TR0N156E_1998_2017-10.txt"}, false); + return TestUtil.getTestDataFileAsserted(testFilePath); + } + + private static File getPirataProduct() throws IOException { + final String testFilePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "pirata-sss", "v1", "2016", "10", "PIRATA_0N35W_sss_2016-10.txt"}, false); + return TestUtil.getTestDataFileAsserted(testFilePath); + } + + private static File getRamaProduct() throws IOException { + final String testFilePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "rama-sss", "v1", "2018", "04", "RAMA_8S805E_sss_2018-04.txt"}, false); + return TestUtil.getTestDataFileAsserted(testFilePath); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSProviderTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSProviderTest.java new file mode 100644 index 000000000..cdf033662 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSProviderTest.java @@ -0,0 +1,36 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class POSProviderTest { + + @Test + public void testInterpolate() { + final POSRecord before = new POSRecord(); + before.date = 1451952099; + before.lon = 28.5f; + before.lat = -10.f; + + final POSRecord after = new POSRecord(); + after.date = 1451952299; + after.lon = 29.5f; + after.lat = -10.4f; + + POSRecord interpolated = POSProvider.interpolate(before, after, 1451952199); + assertEquals(1451952199, interpolated.date); + assertEquals(29.f, interpolated.lon, 1e-8); + assertEquals(-10.2f, interpolated.lat, 1e-8); + + interpolated = POSProvider.interpolate(before, after, 1451952110); + assertEquals(1451952110, interpolated.date); + assertEquals(28.555f, interpolated.lon, 1e-8); + assertEquals(-10.022f, interpolated.lat, 1e-8); + + interpolated = POSProvider.interpolate(before, after, 1451952286); + assertEquals(1451952286, interpolated.date); + assertEquals(29.435f, interpolated.lon, 1e-8); + assertEquals(-10.374f, interpolated.lat, 1e-8); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSProvider_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSProvider_IO_Test.java new file mode 100644 index 000000000..e3c40ae40 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/POSProvider_IO_Test.java @@ -0,0 +1,74 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import com.bc.fiduceo.IOTestRunner; +import com.bc.fiduceo.TestUtil; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +@RunWith(IOTestRunner.class) +public class POSProvider_IO_Test { + + @Test + public void testOpenAndGet_matchTime() throws IOException { + final POSProvider posProvider = new POSProvider(); + final File testFile = TestUtil.getTestDataFileAsserted("insitu/tao-sss/TAO_T2N165E_R_POS.ascii"); + + posProvider.open(testFile, 165.f, 2.f); + + POSRecord posRecord = posProvider.get(1451952099); + assertEquals(1451952099, posRecord.date); + assertEquals(165.15f, posRecord.lon, 1e-8); + assertEquals(2.02f, posRecord.lat, 1e-8); + + posRecord = posProvider.get(1452304908); + assertEquals(1452304908, posRecord.date); + assertEquals(165.14f, posRecord.lon, 1e-8); + assertEquals(2.f, posRecord.lat, 1e-8); + } + + @Test + public void testOpenAndGet_interpolate() throws IOException { + final POSProvider posProvider = new POSProvider(); + final File testFile = TestUtil.getTestDataFileAsserted("insitu/tao-sss/TAO_T2N165E_R_POS.ascii"); + + posProvider.open(testFile,165.f, 2.f); + + POSRecord posRecord = posProvider.get(1451952199); + assertEquals(1451952199, posRecord.date); + assertEquals(165.15f, posRecord.lon, 1e-8); + assertEquals(2.02f, posRecord.lat, 1e-8); + + posRecord = posProvider.get(1533970700); + assertEquals(1533970700, posRecord.date); + assertEquals(164.77195739746094f, posRecord.lon, 1e-8); + assertEquals(2.248743772506714f, posRecord.lat, 1e-8); + } + + @Test + public void testOpenAndGet_noTimeMatch() throws IOException { + final POSProvider posProvider = new POSProvider(); + final File testFile = TestUtil.getTestDataFileAsserted("insitu/tao-sss/TAO_T2N165E_R_POS.ascii"); + + posProvider.open(testFile,165.f, 2.f); + + POSRecord posRecord = posProvider.get(1651606499); + assertEquals(165.f, posRecord.lon, 1e-8); + assertEquals(2.f, posRecord.lat, 1e-8); + } + + @Test + public void testOpenWithNullFileAndGet() { + final POSProvider posProvider = new POSProvider(); + + posProvider.open(null,165.f, 2.f); + + POSRecord posRecord = posProvider.get(1651606499); + assertEquals(165.f, posRecord.lon, 1e-8); + assertEquals(2.f, posRecord.lat, 1e-8); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/TAORecordTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/TAORecordTest.java new file mode 100644 index 000000000..9374bdd86 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/TAORecordTest.java @@ -0,0 +1,28 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TAORecordTest { + + @Test + public void testToString() { + TAORecord taoRecord = new TAORecord(); + taoRecord.date = 1543910594; + taoRecord.lon = "-118.34"; + taoRecord.lat = "51.886"; + taoRecord.SSS = "23.5"; + taoRecord.SST = "24.6"; + taoRecord.AIRT = "25.7"; + taoRecord.RH = "26.8"; + taoRecord.WSPD = "27.9"; + taoRecord.WDIR = "29.0"; + taoRecord.BARO = "30.1"; + taoRecord.RAIN = "31.2"; + taoRecord.Q = "2342345"; + taoRecord.M = "DMDMDMD"; + + assertEquals("1543910594 -118.34 51.886 23.5 24.6 25.7 26.8 27.9 29.0 30.1 31.2 2342345 DMDMDMD", taoRecord.toLine()); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/TaoPreProcessorTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/TaoPreProcessorTest.java new file mode 100644 index 000000000..b1159d012 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/insitu/tao/preproc/TaoPreProcessorTest.java @@ -0,0 +1,17 @@ +package com.bc.fiduceo.reader.insitu.tao.preproc; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TaoPreProcessorTest { + + @Test + public void testToUnixEpoch() { + int unixEpoch = TaoPreProcessor.toUnixEpoch("20190223", "145623"); + assertEquals(1550933783, unixEpoch); + + unixEpoch = TaoPreProcessor.toUnixEpoch("20181204", "080314"); + assertEquals(1543910594, unixEpoch); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReaderTest.java b/core/src/test/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReaderTest.java index a118e8ef0..da89ca4d6 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReaderTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReaderTest.java @@ -2,8 +2,8 @@ import com.bc.fiduceo.reader.Reader; import com.bc.fiduceo.reader.ReaderContext; -import org.esa.s3tbx.dataio.s3.Manifest; -import org.esa.s3tbx.dataio.s3.XfduManifest; +import eu.esa.opt.dataio.s3.Manifest; +import eu.esa.opt.dataio.s3.XfduManifest; import org.junit.Test; import org.w3c.dom.Document; import org.xml.sax.InputSource; diff --git a/core/src/test/java/com/bc/fiduceo/reader/smap/SmapPixelLocatorTest.java b/core/src/test/java/com/bc/fiduceo/reader/smap/SmapPixelLocatorTest.java new file mode 100644 index 000000000..563fcfff4 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/smap/SmapPixelLocatorTest.java @@ -0,0 +1,124 @@ +package com.bc.fiduceo.reader.smap; + +import com.bc.fiduceo.IOTestRunner; +import com.bc.fiduceo.TestUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; +import ucar.nc2.Variable; + +import java.awt.geom.Point2D; +import java.io.File; +import java.io.IOException; + +import static org.hamcrest.Matchers.*; +import static org.hamcrest.MatcherAssert.*; +import static org.junit.Assert.*; + +@RunWith(IOTestRunner.class) +public class SmapPixelLocatorTest { + + private NetcdfFile ncOpened; + + @Before + public void setUp() throws Exception { + final String testFilePath = TestUtil.assembleFileSystemPath(new String[]{"smap-sss", "v05.0", "2018", "02", "04", "RSS_SMAP_SSS_L2C_r16092_20180204T202311_2018035_FNL_V05.0.nc"}, false); + final File file = TestUtil.getTestDataFileAsserted(testFilePath); + ncOpened = NetcdfFiles.open(file.getAbsolutePath()); + } + + @After + public void tearDown() throws Exception { + ncOpened.close(); + } + + @Test + public void testConstructor() throws IOException { + assertNotNull(createLocator()); + } + + @Test + public void testThatGetGeoLocationReturnsANewPoint2DInstanceIfGivenGelolocationIsNull() throws IOException { + final SmapPixelLocator locator = createLocator(); + + Point2D geoLocation = null; + + geoLocation = locator.getGeoLocation(1455.5, 20.5, geoLocation); + assertNotNull(geoLocation); + assertEquals(50.03237915, geoLocation.getX(), 1e-6); + assertEquals(-84.91764832, geoLocation.getY(), 1e-6); + } + + @Test + public void testThatGetGeoLocationReusesAGivenPoint2DInstanceToReturnLonLatValues() throws IOException { + final SmapPixelLocator locator = createLocator(); + + final Point2D geoLocation = new Point2D.Float(); + + final Point2D retVal = locator.getGeoLocation(15.5, 20.5, geoLocation); + assertNotNull(retVal); + assertSame(geoLocation, retVal); + assertEquals(50.1703681, retVal.getX(), 1e-6); + assertEquals(-84.8761215, retVal.getY(), 1e-6); + + final Point2D retVal2 = locator.getGeoLocation(1455.5, 20.5, geoLocation); + assertNotNull(retVal2); + assertSame(geoLocation, retVal2); + assertEquals(50.03237915, retVal2.getX(), 1e-6); + assertEquals(-84.91764832, retVal2.getY(), 1e-6); + } + + @Test + public void testThatGetGeoLocationReturnsNullIfThereIsNoValidGeoposition() throws IOException { + final SmapPixelLocator locator = createLocator(); + + final Point2D geoLocation = locator.getGeoLocation(13.5, 7.5, null); + assertNull(geoLocation); + } + + @Test + public void testThatGetPixelLocationReturns2LocationsIfAvailable() throws IOException { + final SmapPixelLocator locator = createLocator(); + + final Point2D[] pixelLoc = locator.getPixelLocation(50.1703681, -84.8761215); + assertNotNull(pixelLoc); + assertEquals(2, pixelLoc.length); + assertEquals(15.5, pixelLoc[0].getX(), 1e-6); + assertEquals(20.5, pixelLoc[0].getY(), 1e-6); + assertEquals(1454.5, pixelLoc[1].getX(), 1e-6); + assertEquals(20.5, pixelLoc[1].getY(), 1e-6); + } + + @Test + public void testThatGetPixelLocationWorksOnLeftProductBorder() throws IOException { + final SmapPixelLocator locator = createLocator(); + + final Point2D[] pixelLoc = locator.getPixelLocation(53.83325958, -79.87584686); + assertNotNull(pixelLoc); + assertEquals(2, pixelLoc.length); + assertEquals(0.5, pixelLoc[0].getX(), 1e-6); + assertEquals(40.5, pixelLoc[0].getY(), 1e-6); + assertEquals(1440.5, pixelLoc[1].getX(), 1e-6); + assertEquals(40.5, pixelLoc[1].getY(), 1e-6); + } + + @Test + public void testGetPixelLocationReturnsEmptyArrayIfGeoLocationPointsToFillValuesArea() throws IOException { + final SmapPixelLocator locator = createLocator(); + + // points to x: 1135.0 y: 487.0 + final Point2D[] pixelLoc = locator.getPixelLocation(130.125, 31.88); + assertNotNull(pixelLoc); + assertEquals(0, pixelLoc.length); + } + + private SmapPixelLocator createLocator() throws IOException { + final Variable latVar = ncOpened.findVariable("cellat"); + final Variable lonVar = ncOpened.findVariable("cellon"); + final SmapPixelLocator locator = new SmapPixelLocator(lonVar, latVar, 0); + return locator; + } +} \ No newline at end of file diff --git a/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReaderPluginTest_AftLook.java b/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReaderPluginTest_AftLook.java new file mode 100644 index 000000000..80b3acdb6 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReaderPluginTest_AftLook.java @@ -0,0 +1,41 @@ +package com.bc.fiduceo.reader.smap; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.ReaderContext; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class SmapReaderPluginTest_AftLook { + + private SmapReaderPluginAftLook plugin; + + @Before + public void setUp() { + plugin = new SmapReaderPluginAftLook(); + } + + @Test + public void testGetSupportedSensorKey() { + final String[] keys = plugin.getSupportedSensorKeys(); + + assertEquals(1, keys.length); + assertEquals("smap-sss-aft", keys[0]); + } + + @Test + public void testGetDataType() { + assertEquals(DataType.POLAR_ORBITING_SATELLITE, plugin.getDataType()); + } + + @Test + public void testCreateReader() { + final Reader reader = plugin.createReader(new ReaderContext()); + assertNotNull(reader); + assertTrue(reader instanceof SmapReader); + final SmapReader smapReader = (SmapReader) reader; + assertEquals(1, smapReader.lookValue); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReaderPluginTest_ForLook.java b/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReaderPluginTest_ForLook.java new file mode 100644 index 000000000..0be7483cf --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReaderPluginTest_ForLook.java @@ -0,0 +1,43 @@ +package com.bc.fiduceo.reader.smap; + +import com.bc.fiduceo.reader.DataType; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.ReaderContext; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class SmapReaderPluginTest_ForLook { + + private SmapReaderPluginForLook plugin; + + @Before + public void setUp() { + plugin = new SmapReaderPluginForLook(); + } + + @Test + public void testGetSupportedSensorKey() { + final String[] keys = plugin.getSupportedSensorKeys(); + + assertEquals(1, keys.length); + assertEquals("smap-sss-for", keys[0]); + } + + @Test + public void testGetDataType() { + assertEquals(DataType.POLAR_ORBITING_SATELLITE, plugin.getDataType()); + } + + @Test + public void testCreateReader() { + final Reader reader = plugin.createReader(new ReaderContext()); + assertNotNull(reader); + assertTrue(reader instanceof SmapReader); + final SmapReader smapReader = (SmapReader) reader; + assertEquals(0, smapReader.lookValue); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReaderTest.java b/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReaderTest.java new file mode 100644 index 000000000..052a17226 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReaderTest.java @@ -0,0 +1,64 @@ +package com.bc.fiduceo.reader.smap; + +import com.bc.fiduceo.reader.ReaderContext; +import org.junit.Before; +import org.junit.Test; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class SmapReaderTest { + + private SmapReader reader; + + @Before + public void setUp() { + reader = new SmapReader(new ReaderContext(), SmapReader.LOOK.AFT); // empty context sufficient in this test tb 2022-11-17 + } + + @Test + public void testGetRegEx() { + final String expected = "RSS_SMAP_SSS_L2C_r\\d{5}_\\d{8}T\\d{6}_\\d{7}_FNL_V05\\.0\\.nc"; + + final String regEx = reader.getRegEx(); + assertEquals(expected, regEx); + + final Pattern pattern = Pattern.compile(expected); + + Matcher matcher = pattern.matcher("RSS_SMAP_SSS_L2C_r71126_20161001T095140_2016275_FNL_V05.0.nc"); + assertTrue(matcher.matches()); + + matcher = pattern.matcher("RSS_SMAP_SSS_L2C_r16092_20180204T202311_2018035_FNL_V05.0.nc"); + assertTrue(matcher.matches()); + + matcher = pattern.matcher("SM_RE07_MIR_CDF3TD_20171120T000000_20171120T235959_330_001_7.DBL.nc"); + assertFalse(matcher.matches()); + + matcher = pattern.matcher("RSS_WindSat_TB_L1C_r79285_20180429T174238_2018119_V08.0.nc"); + assertFalse(matcher.matches()); + } + + @Test + public void testGetLongitudeVariableName() { + assertEquals("cellon", reader.getLongitudeVariableName()); + } + + @Test + public void testGetLatitudeVariableName() { + assertEquals("cellat", reader.getLatitudeVariableName()); + } + + @Test + public void testExtractYearMonthDayFromFilename() { + int[] ymd = reader.extractYearMonthDayFromFilename("RSS_SMAP_SSS_L2C_r16092_20180224T202311_2018035_FNL_V05.0.nc"); + assertArrayEquals(new int[]{2018, 2, 24}, ymd); + + ymd = reader.extractYearMonthDayFromFilename("RSS_SMAP_SSS_L2C_r71126_20161001T095140_2016275_FNL_V05.0.nc"); + assertArrayEquals(new int[]{2016, 10, 01}, ymd); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReader_IO_Test.java new file mode 100644 index 000000000..8832b8fc9 --- /dev/null +++ b/core/src/test/java/com/bc/fiduceo/reader/smap/SmapReader_IO_Test.java @@ -0,0 +1,450 @@ +package com.bc.fiduceo.reader.smap; + + +import com.bc.fiduceo.IOTestRunner; +import com.bc.fiduceo.NCTestUtils; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.Dimension; +import com.bc.fiduceo.core.Interval; +import com.bc.fiduceo.core.NodeType; +import com.bc.fiduceo.geometry.Geometry; +import com.bc.fiduceo.geometry.GeometryFactory; +import com.bc.fiduceo.geometry.L3TimeAxis; +import com.bc.fiduceo.geometry.Point; +import com.bc.fiduceo.geometry.Polygon; +import com.bc.fiduceo.geometry.TimeAxis; +import com.bc.fiduceo.location.PixelLocator; +import com.bc.fiduceo.reader.AcquisitionInfo; +import com.bc.fiduceo.reader.Reader; +import com.bc.fiduceo.reader.ReaderContext; +import com.bc.fiduceo.reader.time.TimeLocator; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.Array; +import ucar.ma2.ArrayInt; +import ucar.ma2.DataType; +import ucar.ma2.IndexIterator; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; + +import java.awt.geom.Point2D; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import static org.junit.Assert.*; + +@SuppressWarnings("ConstantConditions") +@RunWith(IOTestRunner.class) +public class SmapReader_IO_Test { + + private Reader reader; + + @Before + public void setUp() throws IOException { + final ReaderContext readerContext = new ReaderContext(); + readerContext.setGeometryFactory(new GeometryFactory(GeometryFactory.Type.S2)); + + reader = new SmapReaderPluginAftLook().createReader(readerContext); + } + + @Test + public void testReadAcquisitionInfo() throws IOException { + final File file = getSmapSssFile(); + + try { + reader.open(file); + + final AcquisitionInfo info = reader.read(); + + final Geometry boundingGeometry = info.getBoundingGeometry(); + assertTrue(boundingGeometry instanceof Polygon); + + Point[] coordinates = boundingGeometry.getCoordinates(); + assertEquals(5, coordinates.length); + assertEquals(-179.98402404785156, coordinates[0].getLon(), 1e-8); + assertEquals(-86.61946868896484, coordinates[0].getLat(), 1e-8); + assertEquals(179.99179077148438, coordinates[1].getLon(), 1e-8); + assertEquals(-86.61946868896484, coordinates[1].getLat(), 1e-8); + assertEquals(179.99179077148438, coordinates[2].getLon(), 1e-8); + assertEquals(86.40682220458984, coordinates[2].getLat(), 1e-8); + assertEquals(-179.98402404785156, coordinates[3].getLon(), 1e-8); + assertEquals(86.40682220458984, coordinates[3].getLat(), 1e-8); + assertEquals(-179.98402404785156, coordinates[4].getLon(), 1e-8); + assertEquals(-86.61946868896484, coordinates[4].getLat(), 1e-8); + + final Date sensingStart = info.getSensingStart(); + TestUtil.assertCorrectUTCDate(2018, 2, 4, 20, 23, 11, sensingStart); + final Date sensingStop = info.getSensingStop(); + TestUtil.assertCorrectUTCDate(2018, 2, 4, 22, 4, 56, sensingStop); + + final TimeAxis[] timeAxes = info.getTimeAxes(); + assertEquals(1, timeAxes.length); + final TimeAxis timeAxis = timeAxes[0]; + assertTrue(timeAxis instanceof L3TimeAxis); + final Geometry geometry = timeAxis.getGeometry(); + coordinates = geometry.getCoordinates(); + assertEquals(4, coordinates.length); + assertEquals(-179.98402404785156, coordinates[0].getLon(), 1e-8); + assertEquals(0.0, coordinates[0].getLat(), 1e-8); + assertEquals(179.99179077148438, coordinates[1].getLon(), 1e-8); + assertEquals(0.0, coordinates[1].getLat(), 1e-8); + assertEquals(0.0, coordinates[2].getLon(), 1e-8); + assertEquals(86.40682220458984, coordinates[2].getLat(), 1e-8); + assertEquals(0.0, coordinates[3].getLon(), 1e-8); + assertEquals(-86.61946868896484, coordinates[3].getLat(), 1e-8); + + assertEquals(NodeType.UNDEFINED, info.getNodeType()); + } finally { + reader.close(); + } + } + + @Test + public void testGetProductSize() throws IOException { + final File file = getSmapSssFile(); + + try { + reader.open(file); + + final Dimension productSize = reader.getProductSize(); + assertNotNull(productSize); + assertEquals(1560, productSize.getNx()); + assertEquals(720, productSize.getNy()); + } finally { + reader.close(); + } + } + + @Test + public void testGetPixelLocator() throws IOException { + final File file = getSmapSssFile(); + + try { + reader.open(file); + + final PixelLocator pixelLocator = reader.getPixelLocator(); + assertNotNull(pixelLocator); + + Point2D geoLocation = pixelLocator.getGeoLocation(0.5, 25.5, null); + assertEquals(53.93850708, geoLocation.getX(), 1e-8); + assertEquals(-83.62055206, geoLocation.getY(), 1e-8); + + geoLocation = pixelLocator.getGeoLocation(3119.5, 0.5, null); // x coordinate is outside the product + assertNull(geoLocation); + + geoLocation = pixelLocator.getGeoLocation(20.5, 40.5, null); + assertEquals(48.90038681, geoLocation.getX(), 1e-8); + assertEquals(-79.87638855, geoLocation.getY(), 1e-8); + + geoLocation = pixelLocator.getGeoLocation(780.5, 700.5, null); + assertEquals(218.93360901, geoLocation.getX(), 1e-8); + assertEquals(85.11258698, geoLocation.getY(), 1e-8); + + Point2D[] pixelLocations = pixelLocator.getPixelLocation(53.93850708, -83.62055206); + assertEquals(2, pixelLocations.length); + assertEquals(0.5, pixelLocations[0].getX(), 1e-8); + assertEquals(25.5, pixelLocations[0].getY(), 1e-8); + assertEquals(1440.5, pixelLocations[1].getX(), 1e-8); + assertEquals(25.5, pixelLocations[1].getY(), 1e-8); + + pixelLocations = pixelLocator.getPixelLocation(48.90038681, -79.87638855); + assertEquals(2, pixelLocations.length); + assertEquals(20.5, pixelLocations[0].getX(), 1e-8); + assertEquals(40.5, pixelLocations[0].getY(), 1e-8); + assertEquals(1460.5, pixelLocations[1].getX(), 1e-8); + assertEquals(40.5, pixelLocations[1].getY(), 1e-8); + + pixelLocations = pixelLocator.getPixelLocation(218.93360901, 85.11258698); + assertEquals(1, pixelLocations.length); + assertEquals(780.5, pixelLocations[0].getX(), 1e-8); + assertEquals(700.5, pixelLocations[0].getY(), 1e-8); + + } finally { + reader.close(); + } + } + + @Test + public void testGetSubScenePixelLocator() throws IOException { + final File file = getSmapSssFile(); + + try { + reader.open(file); + + final PixelLocator subScenePixelLocator = reader.getSubScenePixelLocator(null);// geometry is not used here tb 2022-09-29 + final PixelLocator pixelLocator = reader.getPixelLocator(); + + assertSame(pixelLocator, subScenePixelLocator); + } finally { + reader.close(); + } + } + + @Test + public void testGetTimeLocator() throws IOException { + final File file = getSmapSssFile(); + + try { + reader.open(file); + + final TimeLocator timeLocator = reader.getTimeLocator(); + assertNotNull(timeLocator); + + // check fill value areas + assertEquals(-1, timeLocator.getTimeFor(200, 310)); + assertEquals(-1, timeLocator.getTimeFor(600, 310)); + assertEquals(-1, timeLocator.getTimeFor(1300, 310)); + + // check data areas + assertEquals(1517776253000L, timeLocator.getTimeFor(300, 70)); + assertEquals(1517779003000L, timeLocator.getTimeFor(1000, 690)); + assertEquals(1517781791000L, timeLocator.getTimeFor(1450, 50)); + } finally { + reader.close(); + } + } + + @Test + public void testReadAcquisitionTime() throws IOException, InvalidRangeException { + final File file = getSmapSssFile(); + + try { + reader.open(file); + + // read a section that covers a swath border tb 2022-11-28 + final ArrayInt.D2 acquisitionTime = reader.readAcquisitionTime(369, 336, new Interval(5, 3)); + assertEquals(15, acquisitionTime.getSize()); + + NCTestUtils.assertValueAt(1517777339, 0, 0, acquisitionTime); + NCTestUtils.assertValueAt(1517777343, 1, 0, acquisitionTime); + NCTestUtils.assertValueAt(1517777346, 2, 0, acquisitionTime); + NCTestUtils.assertValueAt(1517777349, 3, 0, acquisitionTime); + NCTestUtils.assertValueAt(-2147483647, 4, 0, acquisitionTime); + NCTestUtils.assertValueAt(1517777343, 0, 1, acquisitionTime); + NCTestUtils.assertValueAt(1517777346, 1, 1, acquisitionTime); + NCTestUtils.assertValueAt(1517777350, 2, 1, acquisitionTime); + NCTestUtils.assertValueAt(-2147483647, 3, 1, acquisitionTime); + NCTestUtils.assertValueAt(-2147483647, 4, 1, acquisitionTime); + NCTestUtils.assertValueAt(1517777346, 0, 2, acquisitionTime); + NCTestUtils.assertValueAt(1517777350, 1, 2, acquisitionTime); + NCTestUtils.assertValueAt(1517777353, 2, 2, acquisitionTime); + NCTestUtils.assertValueAt(1517777356, 3, 2, acquisitionTime); + NCTestUtils.assertValueAt(-2147483647, 4, 2, acquisitionTime); + + + } finally { + reader.close(); + } + } + + @Test + public void testGetVariables() throws IOException, InvalidRangeException { + final File file = getSmapSssFile(); + + try { + reader.open(file); + + final List variables = reader.getVariables(); + assertEquals(124, variables.size()); + + Variable variable = variables.get(0); + assertEquals("time_aft", variable.getShortName()); + assertEquals(DataType.DOUBLE, variable.getDataType()); + Attribute attribute = variable.attributes().findAttribute("units"); + assertEquals("seconds since 2000-1-1 0:0:0 0", attribute.getStringValue()); + + variable = variables.get(4); + assertEquals("fland_aft", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + attribute = variable.attributes().findAttribute("add_offset"); + assertEquals(0.f, attribute.getNumericValue().floatValue(), 1e-8); + + variable = variables.get(15); + assertEquals("temp_ant_aft_V", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + attribute = variable.attributes().findAttribute("units"); + assertEquals("Kelvin", attribute.getStringValue()); + + variable = variables.get(26); + assertEquals("sun_beta_aft", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + attribute = variable.attributes().findAttribute("coverage_content_type"); + assertEquals("physicalMeasurement", attribute.getStringValue()); + + variable = variables.get(37); + assertEquals("dtemp_ant_aft_H", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + attribute = variable.attributes().findAttribute("long_name"); + assertEquals("Empirical correction to physical temperature of reflector. Pol basis V,H", attribute.getStringValue()); + + variable = variables.get(48); + assertEquals("ta_gal_ref_aft_Q", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + attribute = variable.attributes().findAttribute("scale_factor"); + assertEquals(1.f, attribute.getNumericValue().floatValue(), 1e-8); + + variable = variables.get(59); + assertEquals("tb_toi_aft_H", variable.getShortName()); + assertEquals(DataType.FLOAT, variable.getDataType()); + attribute = variable.attributes().findAttribute("standard_name"); + assertEquals("brightness_temperature", attribute.getStringValue()); + + variable = variables.get(88); + assertEquals("iqc_flag_aft", variable.getShortName()); + assertEquals(DataType.INT, variable.getDataType()); + attribute = variable.attributes().findAttribute("flag_meaning"); + assertEquals("bit 0 set: no radiometer observation in cell. SSS not retrieved. " + + "bit 1 set: problenm with OI. SSS not retrieved. " + + "bit 2 set: strong land contamination. SSS not retrieved. " + + "bit 3 set: strong sea ice contamination. SSS not retrieved. " + + "bit 4 set: MLE in SSS retrieval algo has not converged. SSS not retrieved. " + + "bit 5 set: sunglint. SSS retrieved. very strong degradation. " + + "bit 6 set: moonglint. SSS retrieved. moderate - strong degradation. " + + "bit 7 set: high reflected galaxy. SSS retrieved. moderate - strong degradation. " + + "bit 8 set: moderate land contamination. SSS retrieved. strong degradation. " + + "bit 9 set: moderate sea ice contamination. SSS retrieved. strong degradation. " + + "bit 10 set: high residual of MLE in SSS retrieval algo. SSS retrieved. strong degradation. " + + "bit 11 set: low SST. SSS retrieved. moderate - strong degradation. " + + "bit 12 set: high wind. SSS retrieved. moderate degradation. " + + "bit 13 set: light land contamination. SSS retrieved. light degradation. not used in ocean target cal. " + + "bit 14 set: light sea ice contamination. SSS retrieved. light - moderate degradation. not used in ocean target cal. " + + "bit 15 set: rain flag. SSS retrieved. possibly light degradation. not used in ocean target cal. " + + "bit 16 set: climatological sea-ice flag set. no AMSR2 data available for sea-ice detection or correction. no SSS retrieved." + , attribute.getStringValue()); + attribute = variable.attributes().findAttribute("coverage_content_type"); + assertEquals("qualityInformation", attribute.getStringValue()); + + variable = variables.get(variables.size() - 1); + assertNotNull(variable); + } finally { + reader.close(); + } + } + + @Test + public void testReadRaw() throws IOException, InvalidRangeException { + final File file = getSmapSssFile(); + + try { + reader.open(file); + + final Array array = reader.readRaw(342, 112, new Interval(3, 3), "sss_smap_40km_unc_comp_sst_aft"); + assertNotNull(array); + assertArrayEquals(new int[]{3, 3}, array.getShape()); + NCTestUtils.assertValueAt(0.2572820484638214, 0, 0, array); + NCTestUtils.assertValueAt(-9999.0, 1, 0, array); + NCTestUtils.assertValueAt(-9999.0, 2, 0, array); + NCTestUtils.assertValueAt(0.28291627764701843, 0, 1, array); + NCTestUtils.assertValueAt(6.475381851196289, 1, 1, array); + NCTestUtils.assertValueAt(-9999.0, 2, 1, array); + NCTestUtils.assertValueAt(-9999.0, 0, 2, array); + NCTestUtils.assertValueAt(-9999.0, 1, 2, array); + NCTestUtils.assertValueAt(0.6615973711013794, 2, 2, array); + } finally { + reader.close(); + } + } + + @Test + public void testReadRaw_data_partially_outside() throws IOException, InvalidRangeException { + final File file = getSmapSssFile(); + + try { + reader.open(file); + + // upper left corner + Array partialOutsideRead = reader.readRaw(0, 6, new Interval(3, 15), "ta_ant_aft_S3"); + assertNotNull(partialOutsideRead); + assertArrayEquals(new int[]{15, 3}, partialOutsideRead.getShape()); + IndexIterator indexIterator = partialOutsideRead.getIndexIterator(); + while (indexIterator.hasNext()) { + Object next = indexIterator.next(); + final int[] currentCounter = indexIterator.getCurrentCounter(); + if (Arrays.equals(new int[]{14, 1}, currentCounter)) { + assertEquals(0.921287477016449, (Float) next, 1e-8); + } else if (Arrays.equals(new int[]{14, 2}, currentCounter)) { + assertEquals(0.6892231106758118, (Float) next, 1e-8); + } else { + assertEquals("Pos: " + Arrays.toString(currentCounter), -9999.0f, (Float) next, 1e-8); + } + } + + // lower border + partialOutsideRead = reader.readRaw(780, 713, new Interval(3, 17), "ta_ant_aft_S3"); + assertNotNull(partialOutsideRead); + assertArrayEquals(new int[]{17, 3}, partialOutsideRead.getShape()); + indexIterator = partialOutsideRead.getIndexIterator(); + while (indexIterator.hasNext()) { + Object next = indexIterator.next(); + final int[] currentCounter = indexIterator.getCurrentCounter(); + if (Arrays.equals(new int[]{0, 0}, currentCounter)) { + assertEquals(0.21593234, (Float) next, 1e-8); + } else if (Arrays.equals(new int[]{0, 2}, currentCounter)) { + assertEquals(0.10643639, (Float) next, 1e-8); + } else { + assertEquals("Pos: " + Arrays.toString(currentCounter), -9999.0f, (Float) next, 1e-8); + } + } + + // right border + partialOutsideRead = reader.readRaw(1549, 37, new Interval(25, 3), "ta_ant_aft_S3"); + assertNotNull(partialOutsideRead); + assertArrayEquals(new int[]{3, 25}, partialOutsideRead.getShape()); + indexIterator = partialOutsideRead.getIndexIterator(); + while (indexIterator.hasNext()) { + Object next = indexIterator.next(); + final int[] currentCounter = indexIterator.getCurrentCounter(); + if (Arrays.equals(new int[]{0, 0}, currentCounter)) { + assertEquals(0.43374702, (Float) next, 1e-8); + } else if (Arrays.equals(new int[]{1, 0}, currentCounter)) { + // Attention. This value cannot be displayed with the Panoply tool. Panoply only displays + // values within the defined value range. The variable "ta_ant" has a defined value range + // from "valid_min = 0.0f" to "valid_max = 330.0f". + // The HDFViewer displays this value. + assertEquals(-0.25885728, (Float) next, 1e-8); + } else if (Arrays.equals(new int[]{2, 0}, currentCounter)) { + assertEquals(0.61770272, (Float) next, 1e-8); + } else { + assertEquals("Pos: " + Arrays.toString(currentCounter), -9999.0f, (Float) next, 1e-8); + } + } + } finally { + reader.close(); + } + } + + @Test + public void testReadScaled() throws IOException, InvalidRangeException { + final File file = getSmapSssFile(); + + try { + reader.open(file); + + // Since SMAP variables apparently have neither an add_offset nor a scale_factor != 1.0, the results are the same as readRaw. + final Array array = reader.readScaled(342, 112, new Interval(3, 3), "sss_smap_40km_unc_comp_sst_aft"); + assertNotNull(array); + assertArrayEquals(new int[]{3, 3}, array.getShape()); + NCTestUtils.assertValueAt(0.2572820484638214, 0, 0, array); + NCTestUtils.assertValueAt(-9999.0, 1, 0, array); + NCTestUtils.assertValueAt(-9999.0, 2, 0, array); + NCTestUtils.assertValueAt(0.28291627764701843, 0, 1, array); + NCTestUtils.assertValueAt(6.475381851196289, 1, 1, array); + NCTestUtils.assertValueAt(-9999.0, 2, 1, array); + NCTestUtils.assertValueAt(-9999.0, 0, 2, array); + NCTestUtils.assertValueAt(-9999.0, 1, 2, array); + NCTestUtils.assertValueAt(0.6615973711013794, 2, 2, array); + } finally { + reader.close(); + } + } + + private File getSmapSssFile() throws IOException { + final String testFilePath = TestUtil.assembleFileSystemPath(new String[]{"smap-sss", "v05.0", "2018", "02", "04", "RSS_SMAP_SSS_L2C_r16092_20180204T202311_2018035_FNL_V05.0.nc"}, false); + return TestUtil.getTestDataFileAsserted(testFilePath); + } +} diff --git a/core/src/test/java/com/bc/fiduceo/reader/avhrr_frac/AVHRR_FRAC_TimeLocatorTest.java b/core/src/test/java/com/bc/fiduceo/reader/time/TimeLocator_StartStopDateTest.java similarity index 76% rename from core/src/test/java/com/bc/fiduceo/reader/avhrr_frac/AVHRR_FRAC_TimeLocatorTest.java rename to core/src/test/java/com/bc/fiduceo/reader/time/TimeLocator_StartStopDateTest.java index 286b274c0..d27101eb1 100644 --- a/core/src/test/java/com/bc/fiduceo/reader/avhrr_frac/AVHRR_FRAC_TimeLocatorTest.java +++ b/core/src/test/java/com/bc/fiduceo/reader/time/TimeLocator_StartStopDateTest.java @@ -1,4 +1,4 @@ -package com.bc.fiduceo.reader.avhrr_frac; +package com.bc.fiduceo.reader.time; import org.junit.Test; @@ -6,14 +6,14 @@ import static org.junit.Assert.assertEquals; -public class AVHRR_FRAC_TimeLocatorTest { +public class TimeLocator_StartStopDateTest { @Test public void testGetTimeFor() { final Date startDate = new Date(1547653120000L); final Date stopDate = new Date(1547653130000L); - final AVHRR_FRAC_TimeLocator timeLocator = new AVHRR_FRAC_TimeLocator(startDate, stopDate, 100); + final TimeLocator_StartStopDate timeLocator = new TimeLocator_StartStopDate(startDate, stopDate, 100); assertEquals(1547653120000L, timeLocator.getTimeFor(3, 0)); assertEquals(1547653120101L, timeLocator.getTimeFor(7, 1)); diff --git a/core/src/test/java/com/bc/fiduceo/util/JDomUtilsTest.java b/core/src/test/java/com/bc/fiduceo/util/JDomUtilsTest.java index 46f34e5df..815b09f06 100644 --- a/core/src/test/java/com/bc/fiduceo/util/JDomUtilsTest.java +++ b/core/src/test/java/com/bc/fiduceo/util/JDomUtilsTest.java @@ -21,9 +21,9 @@ package com.bc.fiduceo.util; import com.bc.fiduceo.core.UseCaseConfig; -import org.jdom.Attribute; -import org.jdom.Document; -import org.jdom.Element; +import org.jdom2.Attribute; +import org.jdom2.Document; +import org.jdom2.Element; import org.junit.Test; import java.util.List; diff --git a/core/src/test/java/com/bc/fiduceo/util/TempFileUtils_IO_Test.java b/core/src/test/java/com/bc/fiduceo/util/TempFileUtils_IO_Test.java index 24482e11c..f5972a952 100644 --- a/core/src/test/java/com/bc/fiduceo/util/TempFileUtils_IO_Test.java +++ b/core/src/test/java/com/bc/fiduceo/util/TempFileUtils_IO_Test.java @@ -27,7 +27,6 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import sun.net.www.protocol.file.FileURLConnection; import java.io.File; import java.io.IOException; diff --git a/core/src/test/resources/com/bc/fiduceo/reader/insitu/generic/XX_config.json b/core/src/test/resources/com/bc/fiduceo/reader/insitu/generic/XX_config.json new file mode 100644 index 000000000..21cf65a48 --- /dev/null +++ b/core/src/test/resources/com/bc/fiduceo/reader/insitu/generic/XX_config.json @@ -0,0 +1,3 @@ +{ + ",name": "XX" +} \ No newline at end of file diff --git a/core/src/test/resources/com/bc/fiduceo/reader/testFile_amsua_l1b.nat b/core/src/test/resources/com/bc/fiduceo/reader/testFile_amsua_l1b.nat new file mode 100644 index 000000000..cfe494a1a Binary files /dev/null and b/core/src/test/resources/com/bc/fiduceo/reader/testFile_amsua_l1b.nat differ diff --git a/environment.yml b/environment.yml index e8bd970d6..2a71afdef 100644 --- a/environment.yml +++ b/environment.yml @@ -3,4 +3,5 @@ channels: - conda-forge dependencies: - python=3.6 - - six=1.16.0 \ No newline at end of file + - six=1.16.0 + - cdsapi=0.6.1 \ No newline at end of file diff --git a/google-s2/pom.xml b/google-s2/pom.xml index 963dbca79..e4866d8e7 100644 --- a/google-s2/pom.xml +++ b/google-s2/pom.xml @@ -8,7 +8,7 @@ com.bc.fiduceo google-s2 - 1.5.8 + 1.6.2 diff --git a/google-s2/src/main/java/com/bc/geometry/s2/S2WKTWriter.java b/google-s2/src/main/java/com/bc/geometry/s2/S2WKTWriter.java index 50326a02c..0cf7d372e 100644 --- a/google-s2/src/main/java/com/bc/geometry/s2/S2WKTWriter.java +++ b/google-s2/src/main/java/com/bc/geometry/s2/S2WKTWriter.java @@ -87,7 +87,7 @@ private static String writePointWkt(S2LatLng geometry) { final StringBuilder builder = new StringBuilder(); builder.append("POINT("); builder.append(geometry.lngDegrees()); - builder.append(","); + builder.append(" "); builder.append(geometry.latDegrees()); builder.append(")"); return builder.toString(); diff --git a/google-s2/src/test/java/com/bc/geometry/s2/S2WKTWriterTest.java b/google-s2/src/test/java/com/bc/geometry/s2/S2WKTWriterTest.java index 6edb4a5d0..3feab10f5 100644 --- a/google-s2/src/test/java/com/bc/geometry/s2/S2WKTWriterTest.java +++ b/google-s2/src/test/java/com/bc/geometry/s2/S2WKTWriterTest.java @@ -62,7 +62,7 @@ public void testWritePoint() { final S2Point s2Point = createS2Point(-18.7, 45.9); final String wkt = S2WKTWriter.write(s2Point); - assertEquals("POINT(-18.7,45.9)", wkt); + assertEquals("POINT(-18.7 45.9)", wkt); } @Test @@ -71,7 +71,7 @@ public void testWritePoint_S2LatLong() { final S2LatLng s2LatLng = new S2LatLng(s2Point); final String wkt = S2WKTWriter.write(s2LatLng); - assertEquals("POINT(-19.800000000000004,46.0)", wkt); + assertEquals("POINT(-19.800000000000004 45.99999999999999)", wkt); } @Test diff --git a/ingestion-tool/pom.xml b/ingestion-tool/pom.xml index 0d914783b..4be54242d 100644 --- a/ingestion-tool/pom.xml +++ b/ingestion-tool/pom.xml @@ -25,7 +25,7 @@ fiduceo-master com.bc.fiduceo - 1.5.8 + 1.6.2 4.0.0 diff --git a/ingestion-tool/src/main/java/com/bc/fiduceo/ingest/IngestionToolMain.java b/ingestion-tool/src/main/java/com/bc/fiduceo/ingest/IngestionToolMain.java index 7bce74d80..f0eaa3f8f 100644 --- a/ingestion-tool/src/main/java/com/bc/fiduceo/ingest/IngestionToolMain.java +++ b/ingestion-tool/src/main/java/com/bc/fiduceo/ingest/IngestionToolMain.java @@ -21,10 +21,7 @@ package com.bc.fiduceo.ingest; import com.bc.fiduceo.log.FiduceoLogger; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; +import org.apache.commons.cli.*; public class IngestionToolMain { @@ -36,7 +33,7 @@ public static void main(String[] args) throws ParseException { return; } - final CommandLineParser parser = new PosixParser(); + final CommandLineParser parser = new DefaultParser(); final CommandLine commandLine = parser.parse(IngestionTool.getOptions(), args); if (commandLine.hasOption("h") || commandLine.hasOption("--help")) { ingestionTool.printUsageTo(System.out); diff --git a/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolIntegrationTest.java b/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolIntegrationTest.java index 23f232e2d..64f8fa378 100644 --- a/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolIntegrationTest.java +++ b/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolIntegrationTest.java @@ -34,8 +34,8 @@ import com.bc.fiduceo.tool.ToolContext; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -49,6 +49,7 @@ import java.sql.SQLException; import java.util.List; +import static com.bc.fiduceo.geometry.GeometryTestUtils.assertSameGeometry; import static org.junit.Assert.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -83,8 +84,11 @@ public void setUp() throws SQLException, IOException { @After public void tearDown() throws SQLException { - storage.clear(); - storage.close(); + if (storage != null) { + storage.clear(); + storage.close(); + storage = null; + } TestUtil.deleteTestDirectory(); } @@ -113,7 +117,7 @@ public void testIngest_errorStopDateBeforeStartDate() throws ParseException, IOE "-start", "2001-004", "-end", "2001-001" }; - final CommandLineParser parser = new PosixParser(); + final CommandLineParser parser = new DefaultParser(); final CommandLine commandLine = parser.parse(IngestionTool.getOptions(), args); final IngestionTool ingestionTool = new IngestionTool(); @@ -179,18 +183,22 @@ public void testIngest_AVHRR_GAC_NOAA10_v013() throws SQLException, ParseExcepti final MultiPolygon multiPolygon = (MultiPolygon) geoBounds; final List polygons = multiPolygon.getPolygons(); assertEquals(2, polygons.size()); - assertEquals(TestData.AVHRR_GAC_N10_GEOMETRIES_v013[0], geometryFactory.format(polygons.get(0))); - assertEquals(TestData.AVHRR_GAC_N10_GEOMETRIES_v013[1], geometryFactory.format(polygons.get(1))); + Geometry expected = geometryFactory.parse(TestData.AVHRR_GAC_N10_GEOMETRIES_v013[0]); + assertSameGeometry(expected, polygons.get(0)); + expected = geometryFactory.parse(TestData.AVHRR_GAC_N10_GEOMETRIES_v013[1]); + assertSameGeometry(expected, polygons.get(1)); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(2, timeAxes.length); TestUtil.assertCorrectUTCDate(1988, 3, 18, 0, 9, 17, 0, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(1988, 3, 18, 1, 6, 16, 0, timeAxes[0].getEndTime()); - assertEquals(TestData.AVHRR_GAC_N10_AXIS_GEOMETRIES_v013[0], geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.AVHRR_GAC_N10_AXIS_GEOMETRIES_v013[0]); + assertSameGeometry(expected, timeAxes[0].getGeometry()); TestUtil.assertCorrectUTCDate(1988, 3, 18, 1, 6, 16, 0, timeAxes[1].getStartTime()); TestUtil.assertCorrectUTCDate(1988, 3, 18, 2, 3, 15, 0, timeAxes[1].getEndTime()); - assertEquals(TestData.AVHRR_GAC_N10_AXIS_GEOMETRIES_v013[1], geometryFactory.format(timeAxes[1].getGeometry())); + expected = geometryFactory.parse(TestData.AVHRR_GAC_N10_AXIS_GEOMETRIES_v013[1]); + assertSameGeometry(expected, timeAxes[1].getGeometry()); } @Test @@ -218,18 +226,22 @@ public void testIngest_AVHRR_GAC_NOAA11_v013() throws SQLException, ParseExcepti final MultiPolygon multiPolygon = (MultiPolygon) geoBounds; final List polygons = multiPolygon.getPolygons(); assertEquals(2, polygons.size()); - assertEquals(TestData.AVHRR_GAC_N11_GEOMETRIES_v013[0], geometryFactory.format(polygons.get(0))); - assertEquals(TestData.AVHRR_GAC_N11_GEOMETRIES_v013[1], geometryFactory.format(polygons.get(1))); + Geometry expected = geometryFactory.parse(TestData.AVHRR_GAC_N11_GEOMETRIES_v013[0]); + assertSameGeometry(expected, polygons.get(0)); + expected = geometryFactory.parse(TestData.AVHRR_GAC_N11_GEOMETRIES_v013[1]); + assertSameGeometry(expected, polygons.get(1)); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(2, timeAxes.length); TestUtil.assertCorrectUTCDate(1991, 5, 9, 7, 51, 46, 0, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(1991, 5, 9, 8, 48, 43, 500, timeAxes[0].getEndTime()); - assertEquals(TestData.AVHRR_GAC_N11_AXIS_GEOMETRIES_v013[0], geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.AVHRR_GAC_N11_AXIS_GEOMETRIES_v013[0]); + assertSameGeometry(expected, timeAxes[0].getGeometry()); TestUtil.assertCorrectUTCDate(1991, 5, 9, 8, 48, 43, 500, timeAxes[1].getStartTime()); TestUtil.assertCorrectUTCDate(1991, 5, 9, 9, 45, 41, 0, timeAxes[1].getEndTime()); - assertEquals(TestData.AVHRR_GAC_N11_AXIS_GEOMETRIES_v013[1], geometryFactory.format(timeAxes[1].getGeometry())); + expected = geometryFactory.parse(TestData.AVHRR_GAC_N11_AXIS_GEOMETRIES_v013[1]); + assertSameGeometry(expected, timeAxes[1].getGeometry()); } @Test @@ -257,18 +269,22 @@ public void testIngest_AVHRR_GAC_NOAA17_v014() throws SQLException, ParseExcepti final MultiPolygon multiPolygon = (MultiPolygon) geoBounds; final List polygons = multiPolygon.getPolygons(); assertEquals(2, polygons.size()); - assertEquals(TestData.AVHRR_GAC_N17_GEOMETRIES_v014_CSPP[0], geometryFactory.format(polygons.get(0))); - assertEquals(TestData.AVHRR_GAC_N17_GEOMETRIES_v014_CSPP[1], geometryFactory.format(polygons.get(1))); + Geometry expected = geometryFactory.parse(TestData.AVHRR_GAC_N17_GEOMETRIES_v014_CSPP[0]); + assertSameGeometry(expected, polygons.get(0)); + expected = geometryFactory.parse(TestData.AVHRR_GAC_N17_GEOMETRIES_v014_CSPP[1]); + assertSameGeometry(expected, polygons.get(1)); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(2, timeAxes.length); TestUtil.assertCorrectUTCDate(2009, 10, 25, 8, 7, 39, 0, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(2009, 10, 25, 9, 4, 9, 0, timeAxes[0].getEndTime()); - assertEquals(TestData.AVHRR_GAC_N17_AXIS_GEOMETRIES_v014_CSPP[0], geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.AVHRR_GAC_N17_AXIS_GEOMETRIES_v014_CSPP[0]); + assertSameGeometry(expected, timeAxes[0].getGeometry()); TestUtil.assertCorrectUTCDate(2009, 10, 25, 9, 4, 9, 0, timeAxes[1].getStartTime()); TestUtil.assertCorrectUTCDate(2009, 10, 25, 10, 0, 39, 0, timeAxes[1].getEndTime()); - assertEquals(TestData.AVHRR_GAC_N17_AXIS_GEOMETRIES_v014_CSPP[1], geometryFactory.format(timeAxes[1].getGeometry())); + expected = geometryFactory.parse(TestData.AVHRR_GAC_N17_AXIS_GEOMETRIES_v014_CSPP[1]); + assertSameGeometry(expected, timeAxes[1].getGeometry()); } @Test @@ -295,18 +311,22 @@ public void testIngest_AMSUB_NOAA15() throws SQLException, ParseException { final MultiPolygon multiPolygon = (MultiPolygon) geoBounds; final List polygons = multiPolygon.getPolygons(); assertEquals(2, polygons.size()); - assertEquals(TestData.AMSUB_N15_GEOMETRIES[0], geometryFactory.format(polygons.get(0))); - assertEquals(TestData.AMSUB_N15_GEOMETRIES[1], geometryFactory.format(polygons.get(1))); + Geometry expected = geometryFactory.parse(TestData.AMSUB_N15_GEOMETRIES[0]); + assertSameGeometry(expected, polygons.get(0)); + expected = geometryFactory.parse(TestData.AMSUB_N15_GEOMETRIES[1]); + assertSameGeometry(expected, polygons.get(1)); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(2, timeAxes.length); TestUtil.assertCorrectUTCDate(2007, 8, 22, 16, 40, 37, 120, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(2007, 8, 22, 17, 32, 45, 119, timeAxes[0].getEndTime()); - assertEquals(TestData.AMSUB_N15_AXIS_GEOMETRIES[0], geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.AMSUB_N15_AXIS_GEOMETRIES[0]); + assertSameGeometry(expected, timeAxes[0].getGeometry()); TestUtil.assertCorrectUTCDate(2007, 8, 22, 17, 32, 45, 119, timeAxes[1].getStartTime()); TestUtil.assertCorrectUTCDate(2007, 8, 22, 18, 24, 53, 119, timeAxes[1].getEndTime()); - assertEquals(TestData.AMSUB_N15_AXIS_GEOMETRIES[1], geometryFactory.format(timeAxes[1].getGeometry())); + expected = geometryFactory.parse(TestData.AMSUB_N15_AXIS_GEOMETRIES[1]); + assertSameGeometry(expected, timeAxes[1].getGeometry()); } @Test @@ -344,18 +364,67 @@ public void testIngest_MHS_NOAA18() throws SQLException, ParseException { final MultiPolygon multiPolygon = (MultiPolygon) geoBounds; final List polygons = multiPolygon.getPolygons(); assertEquals(2, polygons.size()); - assertEquals(TestData.MHS_N18_GEOMETRIES[0], geometryFactory.format(polygons.get(0))); - assertEquals(TestData.MHS_N18_GEOMETRIES[1], geometryFactory.format(polygons.get(1))); + + Geometry expected = geometryFactory.parse(TestData.MHS_N18_GEOMETRIES[0]); + assertSameGeometry(expected, polygons.get(0)); + expected = geometryFactory.parse(TestData.MHS_N18_GEOMETRIES[1]); + assertSameGeometry(expected, polygons.get(1)); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(2, timeAxes.length); TestUtil.assertCorrectUTCDate(2007, 8, 22, 11, 51, 27, 277, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(2007, 8, 22, 12, 44, 29, 943, timeAxes[0].getEndTime()); - assertEquals(TestData.MHS_N18_AXIS_GEOMETRIES[0], geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.MHS_N18_AXIS_GEOMETRIES[0]); + assertSameGeometry(expected, timeAxes[0].getGeometry()); TestUtil.assertCorrectUTCDate(2007, 8, 22, 12, 44, 29, 943, timeAxes[1].getStartTime()); TestUtil.assertCorrectUTCDate(2007, 8, 22, 13, 37, 32, 610, timeAxes[1].getEndTime()); - assertEquals(TestData.MHS_N18_AXIS_GEOMETRIES[1], geometryFactory.format(timeAxes[1].getGeometry())); + expected = geometryFactory.parse(TestData.MHS_N18_AXIS_GEOMETRIES[1]); + assertSameGeometry(expected, timeAxes[1].getGeometry()); + } + + @Test + public void testIngest_MHS_L1B_MC() throws SQLException, ParseException { + final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-s", "mhs-mc-l1b", "-start", "2025-232", "-end", "2025-232", "-v", "v10"}; + + IngestionToolMain.main(args); + final List satelliteObservations = storage.get(); + assertEquals(1, satelliteObservations.size()); + + final SatelliteObservation observation = getSatelliteObservation("MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat", satelliteObservations); + TestUtil.assertCorrectUTCDate(2025, 8, 20, 6, 3, 50, 0, observation.getStartTime()); + TestUtil.assertCorrectUTCDate(2025, 8, 20, 7, 45, 50, 0, observation.getStopTime()); + assertEquals("mhs-mc-l1b", observation.getSensor().getName()); + + final String expectedPath = TestUtil.assembleFileSystemPath(new String[]{"mhs-mc-l1b", "v10", "2025", "08", "20", "MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"}, false); + assertEquals(expectedPath, observation.getDataFilePath().toString()); + + assertEquals(NodeType.UNDEFINED, observation.getNodeType()); + assertEquals("v10", observation.getVersion()); + + final Geometry geoBounds = observation.getGeoBounds(); + assertTrue(geoBounds instanceof MultiPolygon); + final MultiPolygon multiPolygon = (MultiPolygon) geoBounds; + final List polygons = multiPolygon.getPolygons(); + assertEquals(2, polygons.size()); + + Geometry expected = geometryFactory.parse(TestData.MHS_MC_GEOMETRIES[0]); + assertSameGeometry(expected, polygons.get(0)); + + expected = geometryFactory.parse(TestData.MHS_MC_GEOMETRIES[1]); + assertSameGeometry(expected, polygons.get(1)); + + final TimeAxis[] timeAxes = observation.getTimeAxes(); + assertEquals(2, timeAxes.length); + TestUtil.assertCorrectUTCDate(2025, 8, 20, 6, 3, 50, 0, timeAxes[0].getStartTime()); + TestUtil.assertCorrectUTCDate(2025, 8, 20, 6, 54, 50, 0, timeAxes[0].getEndTime()); + expected = geometryFactory.parse(TestData.MHS_MC_AXIS_GEOMETRIES[0]); + assertSameGeometry(expected, timeAxes[0].getGeometry()); + + TestUtil.assertCorrectUTCDate(2025, 8, 20, 6, 54, 50, 0, timeAxes[1].getStartTime()); + TestUtil.assertCorrectUTCDate(2025, 8, 20, 7, 45, 50, 0, timeAxes[1].getEndTime()); + expected = geometryFactory.parse(TestData.MHS_MC_AXIS_GEOMETRIES[1]); + assertSameGeometry(expected, timeAxes[1].getGeometry()); } @Test @@ -384,18 +453,22 @@ public void testIngest_HIRS_TIROSN() throws SQLException, ParseException { final List polygons = multiPolygon.getPolygons(); assertEquals(2, polygons.size()); - assertEquals(TestData.HIRS_TN_GEOMETRIES[0], geometryFactory.format(polygons.get(0))); - assertEquals(TestData.HIRS_TN_GEOMETRIES[1], geometryFactory.format(polygons.get(1))); + Geometry expected = geometryFactory.parse(TestData.HIRS_TN_GEOMETRIES[0]); + assertSameGeometry(expected, polygons.get(0)); + expected = geometryFactory.parse(TestData.HIRS_TN_GEOMETRIES[1]); + assertSameGeometry(expected, polygons.get(1)); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(2, timeAxes.length); TestUtil.assertCorrectUTCDate(1979, 10, 14, 16, 23, 59, 0, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(1979, 10, 14, 17, 15, 46, 0, timeAxes[0].getEndTime()); - assertEquals(TestData.HIRS_TN_AXIS_GEOMETRIES[0], geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.HIRS_TN_AXIS_GEOMETRIES[0]); + assertSameGeometry(expected, timeAxes[0].getGeometry()); TestUtil.assertCorrectUTCDate(1979, 10, 14, 17, 15, 46, 0, timeAxes[1].getStartTime()); TestUtil.assertCorrectUTCDate(1979, 10, 14, 18, 7, 33, 0, timeAxes[1].getEndTime()); - assertEquals(TestData.HIRS_TN_AXIS_GEOMETRIES[1], geometryFactory.format(timeAxes[1].getGeometry())); + expected = geometryFactory.parse(TestData.HIRS_TN_AXIS_GEOMETRIES[1]); + assertSameGeometry(expected, timeAxes[1].getGeometry()); } @Test @@ -424,18 +497,22 @@ public void testIngest_HIRS_NOAA10() throws SQLException, ParseException { final List polygons = multiPolygon.getPolygons(); assertEquals(2, polygons.size()); - assertEquals(TestData.HIRS_N10_GEOMETRIES[0], geometryFactory.format(polygons.get(0))); - assertEquals(TestData.HIRS_N10_GEOMETRIES[1], geometryFactory.format(polygons.get(1))); + Geometry expected = geometryFactory.parse(TestData.HIRS_N10_GEOMETRIES[0]); + assertSameGeometry(expected, polygons.get(0)); + expected = geometryFactory.parse(TestData.HIRS_N10_GEOMETRIES[1]); + assertSameGeometry(expected, polygons.get(1)); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(2, timeAxes.length); TestUtil.assertCorrectUTCDate(1989, 3, 17, 6, 12, 16, 0, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(1989, 3, 17, 7, 7, 9, 0, timeAxes[0].getEndTime()); - assertEquals(TestData.HIRS_N10_AXIS_GEOMETRIES[0], geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.HIRS_N10_AXIS_GEOMETRIES[0]); + assertSameGeometry(expected, timeAxes[0].getGeometry()); TestUtil.assertCorrectUTCDate(1989, 3, 17, 7, 7, 9, 0, timeAxes[1].getStartTime()); TestUtil.assertCorrectUTCDate(1989, 3, 17, 8, 2, 2, 0, timeAxes[1].getEndTime()); - assertEquals(TestData.HIRS_N10_AXIS_GEOMETRIES[1], geometryFactory.format(timeAxes[1].getGeometry())); + expected = geometryFactory.parse(TestData.HIRS_N10_AXIS_GEOMETRIES[1]); + assertSameGeometry(expected, timeAxes[1].getGeometry()); } @Test @@ -464,18 +541,22 @@ public void testIngest_HIRS_METOPA() throws SQLException, ParseException { final List polygons = multiPolygon.getPolygons(); assertEquals(2, polygons.size()); - assertEquals(TestData.HIRS_MA_GEOMETRIES[0], geometryFactory.format(polygons.get(0))); - assertEquals(TestData.HIRS_MA_GEOMETRIES[1], geometryFactory.format(polygons.get(1))); + Geometry expected = geometryFactory.parse(TestData.HIRS_MA_GEOMETRIES[0]); + assertSameGeometry(expected, polygons.get(0)); + expected = geometryFactory.parse(TestData.HIRS_MA_GEOMETRIES[1]); + assertSameGeometry(expected, polygons.get(1)); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(2, timeAxes.length); TestUtil.assertCorrectUTCDate(2011, 8, 23, 16, 41, 52, 0, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(2011, 8, 23, 17, 32, 16, 0, timeAxes[0].getEndTime()); - assertEquals(TestData.HIRS_MA_AXIS_GEOMETRIES[0], geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.HIRS_MA_AXIS_GEOMETRIES[0]); + assertSameGeometry(expected, timeAxes[0].getGeometry()); TestUtil.assertCorrectUTCDate(2011, 8, 23, 17, 32, 16, 0, timeAxes[1].getStartTime()); TestUtil.assertCorrectUTCDate(2011, 8, 23, 18, 22, 40, 0, timeAxes[1].getEndTime()); - assertEquals(TestData.HIRS_MA_AXIS_GEOMETRIES[1], geometryFactory.format(timeAxes[1].getGeometry())); + expected = geometryFactory.parse(TestData.HIRS_MA_AXIS_GEOMETRIES[1]); + assertSameGeometry(expected, timeAxes[1].getGeometry()); } @Test @@ -501,13 +582,15 @@ public void testIngest_ATSR1() throws SQLException, ParseException { final Geometry geoBounds = observation.getGeoBounds(); assertTrue(geoBounds instanceof Polygon); - assertEquals(TestData.ATSR1_GEOMETRY, geometryFactory.format(geoBounds)); + Geometry expected = geometryFactory.parse(TestData.ATSR1_GEOMETRY); + assertSameGeometry(expected, geoBounds); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(1, timeAxes.length); TestUtil.assertCorrectUTCDate(1993, 8, 5, 21, 0, 30, 240, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(1993, 8, 5, 22, 41, 8, 490, timeAxes[0].getEndTime()); - assertEquals(TestData.ATSR1_AXIS_GEOMETRY, geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.ATSR1_AXIS_GEOMETRY); + assertSameGeometry(expected, timeAxes[0].getGeometry()); } @Test @@ -533,13 +616,15 @@ public void testIngest_ATSR2() throws SQLException, ParseException { final Geometry geoBounds = observation.getGeoBounds(); assertTrue(geoBounds instanceof Polygon); - assertEquals(TestData.ATSR2_GEOMETRY, geometryFactory.format(geoBounds)); + Geometry expected = geometryFactory.parse(TestData.ATSR2_GEOMETRY); + assertSameGeometry(expected, geoBounds); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(1, timeAxes.length); TestUtil.assertCorrectUTCDate(1998, 4, 24, 5, 57, 54, 720, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(1998, 4, 24, 7, 38, 32, 970, timeAxes[0].getEndTime()); - assertEquals(TestData.ATSR2_AXIS_GEOMETRY, geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.ATSR2_AXIS_GEOMETRY); + assertSameGeometry(expected, timeAxes[0].getGeometry()); } @Test @@ -565,13 +650,15 @@ public void testIngest_AATSR() throws SQLException, ParseException { final Geometry geoBounds = observation.getGeoBounds(); assertTrue(geoBounds instanceof Polygon); - assertEquals(TestData.AATSR_GEOMETRY, geometryFactory.format(geoBounds)); + Geometry expected = geometryFactory.parse(TestData.AATSR_GEOMETRY); + assertSameGeometry(expected, geoBounds); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(1, timeAxes.length); TestUtil.assertCorrectUTCDate(2006, 2, 15, 7, 8, 52, 812, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(2006, 2, 15, 8, 57, 40, 662, timeAxes[0].getEndTime()); - assertEquals(TestData.AATSR_AXIS_GEOMETRY, geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.AATSR_AXIS_GEOMETRY); + assertSameGeometry(expected, timeAxes[0].getGeometry()); } @Test @@ -597,13 +684,15 @@ public void testIngest_AMSRE() throws SQLException, ParseException { final Geometry geoBounds = observation.getGeoBounds(); assertTrue(geoBounds instanceof Polygon); - assertEquals(TestData.AMSRE_GEOMETRY, geometryFactory.format(geoBounds)); + Geometry expected = geometryFactory.parse(TestData.AMSRE_GEOMETRY); + assertSameGeometry(expected, geoBounds); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(1, timeAxes.length); TestUtil.assertCorrectUTCDate(2005, 2, 17, 5, 36, 6, 0, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(2005, 2, 17, 6, 25, 56, 0, timeAxes[0].getEndTime()); - assertEquals(TestData.AMSRE_AXIS_GEOMETRY, geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.AMSRE_AXIS_GEOMETRY); + assertSameGeometry(expected, timeAxes[0].getGeometry()); } @Test @@ -628,13 +717,16 @@ public void testIngest_SSMT2() throws SQLException, ParseException { final Geometry geoBounds = observation.getGeoBounds(); assertTrue(geoBounds instanceof Polygon); - assertEquals(TestData.SSMT2_GEOMETRY, geometryFactory.format(geoBounds)); + + Geometry expected = geometryFactory.parse(TestData.SSMT2_GEOMETRY); + assertSameGeometry(expected, geoBounds); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(1, timeAxes.length); TestUtil.assertCorrectUTCDate(2001, 6, 14, 12, 29, 4, 0, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(2001, 6, 14, 14, 10, 58, 0, timeAxes[0].getEndTime()); - assertEquals(TestData.SSMT2_AXIS_GEOMETRY, geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.SSMT2_AXIS_GEOMETRY); + assertSameGeometry(expected, timeAxes[0].getGeometry()); } @Test @@ -738,18 +830,22 @@ public void testIngest_IASI_MA() throws SQLException, ParseException { final MultiPolygon multiPolygon = (MultiPolygon) geoBounds; final List polygons = multiPolygon.getPolygons(); assertEquals(2, polygons.size()); - assertEquals(TestData.IASI_MA_GEOMETRIES[0], geometryFactory.format(polygons.get(0))); - assertEquals(TestData.IASI_MA_GEOMETRIES[1], geometryFactory.format(polygons.get(1))); + Geometry expected = geometryFactory.parse(TestData.IASI_MA_GEOMETRIES[0]); + assertSameGeometry(expected, polygons.get(0)); + expected = geometryFactory.parse(TestData.IASI_MA_GEOMETRIES[1]); + assertSameGeometry(expected, polygons.get(1)); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(2, timeAxes.length); TestUtil.assertCorrectUTCDate(2016, 1, 1, 12, 47, 54, 870, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(2016, 1, 1, 13, 37, 26, 642, timeAxes[0].getEndTime()); - assertEquals(TestData.IASI_MA_AXIS_GEOMETRIES[0], geometryFactory.format(timeAxes[0].getGeometry())); + expected = geometryFactory.parse(TestData.IASI_MA_AXIS_GEOMETRIES[0]); + assertSameGeometry(expected, timeAxes[0].getGeometry()); TestUtil.assertCorrectUTCDate(2016, 1, 1, 13, 37, 26, 642, timeAxes[1].getStartTime()); TestUtil.assertCorrectUTCDate(2016, 1, 1, 14, 26, 58, 414, timeAxes[1].getEndTime()); - assertEquals(TestData.IASI_MA_AXIS_GEOMETRIES[1], geometryFactory.format(timeAxes[1].getGeometry())); + expected = geometryFactory.parse(TestData.IASI_MA_AXIS_GEOMETRIES[1]); + assertSameGeometry(expected, timeAxes[1].getGeometry()); } @Test @@ -772,16 +868,18 @@ public void testIngest_MYD06_AQUA() throws SQLException, ParseException { assertEquals(NodeType.UNDEFINED, observation.getNodeType()); assertEquals("v006", observation.getVersion()); - final Geometry geoBounds = observation.getGeoBounds(); - assertTrue(geoBounds instanceof Polygon); + final Geometry actualBounds = observation.getGeoBounds(); + assertTrue(actualBounds instanceof Polygon); - assertEquals(TestData.MYD06_AQUA_GEOMETRY, geometryFactory.format(geoBounds)); + Geometry expectedBounds = geometryFactory.parse(TestData.MYD06_AQUA_GEOMETRY); + assertSameGeometry(expectedBounds, actualBounds); final TimeAxis[] timeAxes = observation.getTimeAxes(); assertEquals(1, timeAxes.length); TestUtil.assertCorrectUTCDate(2009, 5, 13, 10, 35, 0, 0, timeAxes[0].getStartTime()); TestUtil.assertCorrectUTCDate(2009, 5, 13, 10, 40, 0, 0, timeAxes[0].getEndTime()); - assertEquals(TestData.MYD06_AQUA_AXIS_GEOMETRY, geometryFactory.format(timeAxes[0].getGeometry())); + expectedBounds = geometryFactory.parse(TestData.MYD06_AQUA_AXIS_GEOMETRY); + assertSameGeometry(expectedBounds, timeAxes[0].getGeometry()); } @Test @@ -953,8 +1051,10 @@ public void testIngest_MY021KM() throws SQLException, ParseException { assertEquals("v61", observation.getVersion()); final Geometry geoBounds = observation.getGeoBounds(); - assertEquals(TestData.MYD012KM_AQ_GEOMETRY, geometryFactory.format(geoBounds)); - assertEquals(TestData.MYD021KM_AQ_AXIS_GEOMETRY, geometryFactory.format(observation.getTimeAxes()[0].getGeometry())); + Geometry expected = geometryFactory.parse(TestData.MYD012KM_AQ_GEOMETRY); + assertSameGeometry(expected, geoBounds); + expected = geometryFactory.parse(TestData.MYD021KM_AQ_AXIS_GEOMETRY); + assertSameGeometry(expected, observation.getTimeAxes()[0].getGeometry()); } @Test @@ -978,8 +1078,10 @@ public void testIngest_MOD35() throws SQLException, ParseException { assertEquals("v61", observation.getVersion()); final Geometry geoBounds = observation.getGeoBounds(); - assertEquals(TestData.MOD35_TE_GEOMETRY, geometryFactory.format(geoBounds)); - assertEquals(TestData.MOD35_TE_AXIS_GEOMETRY, geometryFactory.format(observation.getTimeAxes()[0].getGeometry())); + Geometry expected = geometryFactory.parse(TestData.MOD35_TE_GEOMETRY); + assertSameGeometry(expected, geoBounds); + expected = geometryFactory.parse(TestData.MOD35_TE_AXIS_GEOMETRY); + assertSameGeometry(expected, observation.getTimeAxes()[0].getGeometry()); } @Test @@ -1003,8 +1105,10 @@ public void testIngest_AVHRR_FRAC_MB() throws SQLException, ParseException { assertEquals("v1", observation.getVersion()); final Geometry geoBounds = observation.getGeoBounds(); - assertEquals(TestData.AVHRR_FRAC_MB_GEOMETRY, geometryFactory.format(geoBounds)); - assertEquals(TestData.AVHRR_FRAC_MB_AXIS_GEOMETRY, geometryFactory.format(observation.getTimeAxes()[0].getGeometry())); + Geometry expected = geometryFactory.parse(TestData.AVHRR_FRAC_MB_GEOMETRY); + assertSameGeometry(expected, geoBounds); + expected = geometryFactory.parse(TestData.AVHRR_FRAC_MB_AXIS_GEOMETRY); + assertSameGeometry(expected, observation.getTimeAxes()[0].getGeometry()); } @Test @@ -1091,6 +1195,8 @@ public void testIngest_windsat_coriolis() throws SQLException, ParseException { final Geometry geoBounds = observation.getGeoBounds(); assertEquals("POLYGON((-179.93750000000003 -89.9375,179.93750000000003 -89.9375,179.93750000000003 89.9375,-179.93750000000003 89.9375,-179.93750000000003 -89.9375))", geometryFactory.format(geoBounds)); + final Geometry intersection = geoBounds.getIntersection(geometryFactory.createPoint(0, 0)); + assertEquals(geometryFactory.createPoint(0, 0), intersection); final TimeAxis timeAxis = observation.getTimeAxes()[0]; assertTrue(timeAxis instanceof L3TimeAxis); assertEquals("MULTILINESTRING((-179.93750000000003 0.0,179.93750000000003 0.0),(0.0 89.9375,0.0 -89.9375))", geometryFactory.format(timeAxis.getGeometry())); @@ -1118,6 +1224,108 @@ public void testIngest_ndbc_standard_meteo() throws SQLException, ParseException assertNull(observation.getTimeAxes()); } + @Test + public void testIngest_gbov() throws SQLException, ParseException { + final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-s", "gbov", "-start", "2016-156", "-end", "2016-156", "-v", "v1"}; + + IngestionToolMain.main(args); + + final List observations = storage.get(); + assertEquals(2, observations.size()); + + SatelliteObservation observation = getSatelliteObservation("GBOV__Bartlett--Experimental--Forest__BART_047__20160601T000000Z__20160628T000000Z.csv", observations); + TestUtil.assertCorrectUTCDate(2016, 6, 1, 0, 0, 0, observation.getStartTime()); + TestUtil.assertCorrectUTCDate(2016, 6, 28, 0, 0, 0, observation.getStopTime()); + + assertEquals("gbov", observation.getSensor().getName()); + assertEquals("v1", observation.getVersion()); + assertEquals(NodeType.UNDEFINED, observation.getNodeType()); + + assertNull(observation.getGeoBounds()); + assertNull(observation.getTimeAxes()); + + observation = getSatelliteObservation("GBOV__Barrow__Barrow__20160601T000000Z__20160630T235900Z.csv", observations); + TestUtil.assertCorrectUTCDate(2016, 6, 1, 0, 0, 0, observation.getStartTime()); + TestUtil.assertCorrectUTCDate(2016, 6, 30, 23, 59, 0, observation.getStopTime()); + + assertEquals("gbov", observation.getSensor().getName()); + assertEquals("v1", observation.getVersion()); + assertEquals(NodeType.UNDEFINED, observation.getNodeType()); + + assertNull(observation.getGeoBounds()); + assertNull(observation.getTimeAxes()); + } + + @Test + public void testIngest_TAO() throws SQLException, ParseException { + final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-s", "tao-sss", "-start", "2017-275", "-end", "2017-275", "-v", "v1"}; + + IngestionToolMain.main(args); + + final List observations = storage.get(); + assertEquals(1, observations.size()); + + final SatelliteObservation observation = getSatelliteObservation("TRITON_TR0N156E_1998_2017-10.txt", observations); + TestUtil.assertCorrectUTCDate(2017, 10, 1, 12, 0, 0, observation.getStartTime()); + TestUtil.assertCorrectUTCDate(2017, 10, 31, 12, 0, 0, observation.getStopTime()); + + assertEquals("tao-sss", observation.getSensor().getName()); + assertEquals("v1", observation.getVersion()); + assertEquals(NodeType.UNDEFINED, observation.getNodeType()); + + assertNull(observation.getGeoBounds()); + assertNull(observation.getTimeAxes()); + } + + @Test + public void testIngest_PIRATA() throws SQLException, ParseException { + final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-s", "pirata-sss", "-start", "2016-275", "-end", "2016-275", "-v", "v1"}; + + IngestionToolMain.main(args); + + final List observations = storage.get(); + assertEquals(1, observations.size()); + + final SatelliteObservation observation = getSatelliteObservation("PIRATA_0N35W_sss_2016-10.txt", observations); + TestUtil.assertCorrectUTCDate(2016, 10, 1, 0, 0, 0, observation.getStartTime()); + TestUtil.assertCorrectUTCDate(2016, 10, 31, 23, 0, 0, observation.getStopTime()); + + assertEquals("pirata-sss", observation.getSensor().getName()); + assertEquals("v1", observation.getVersion()); + assertEquals(NodeType.UNDEFINED, observation.getNodeType()); + + assertNull(observation.getGeoBounds()); + assertNull(observation.getTimeAxes()); + } + + @Test + public void testIngest_smap_sss__for_look() throws SQLException, ParseException { + final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-s", "smap-sss-for", "-start", "2018-031", "-end", "2018-040", "-v", "v05.0"}; + + IngestionToolMain.main(args); + + final List observations = storage.get(); + assertEquals(2, observations.size()); + + final SatelliteObservation observation = getSatelliteObservation("RSS_SMAP_SSS_L2C_r16092_20180204T202311_2018035_FNL_V05.0.nc", observations); + TestUtil.assertCorrectUTCDate(2018, 2, 4, 20, 23, 11, observation.getStartTime()); + TestUtil.assertCorrectUTCDate(2018, 2, 4, 22, 4, 56, observation.getStopTime()); + + assertEquals("smap-sss-for", observation.getSensor().getName()); + assertEquals("v05.0", observation.getVersion()); + assertEquals(NodeType.UNDEFINED, observation.getNodeType()); + + final Geometry geoBounds = observation.getGeoBounds(); + assertEquals("POLYGON((-179.9690856933594 -86.61936950683594,179.9915313720703 -86.61936950683594,179.9915313720703 86.40787506103517,-179.9690856933594 86.40787506103517,-179.9690856933594 -86.61936950683594))", + geometryFactory.format(geoBounds)); + final Geometry intersection = geoBounds.getIntersection(geometryFactory.createPoint(0, 0)); + assertEquals(geometryFactory.createPoint(0, 0), intersection); + final TimeAxis timeAxis = observation.getTimeAxes()[0]; + assertTrue(timeAxis instanceof L3TimeAxis); + assertEquals("MULTILINESTRING((-179.9690856933594 0.0,179.9915313720703 0.0),(0.0 86.40787506103517,0.0 -86.61936950683594))", geometryFactory.format(timeAxis.getGeometry())); + + } + private void callMainAndValidateSystemOutput(String[] args, boolean errorOutputExpected) throws ParseException { final ByteArrayOutputStream expected = new ByteArrayOutputStream(); new IngestionTool().printUsageTo(expected); diff --git a/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolTest.java b/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolTest.java index f086dfc2e..8493f3ca4 100644 --- a/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolTest.java +++ b/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolTest.java @@ -59,7 +59,7 @@ public void testPrintUsageTo() { ingestionTool.printUsageTo(outputStream); - assertEquals("ingestion-tool version 1.5.8" + ls + + assertEquals("ingestion-tool version 1.6.2" + ls + ls + "usage: ingestion-tool " + ls + "Valid options are:" + ls + diff --git a/matchup-tool/pom.xml b/matchup-tool/pom.xml index 4a2cb9ec1..5426497ae 100644 --- a/matchup-tool/pom.xml +++ b/matchup-tool/pom.xml @@ -25,7 +25,7 @@ fiduceo-master com.bc.fiduceo - 1.5.8 + 1.6.2 4.0.0 @@ -62,7 +62,7 @@ org.hamcrest - hamcrest-all + hamcrest test diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/MatchupToolMain.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/MatchupToolMain.java index cc4f7d5c9..cd48e1bc3 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/MatchupToolMain.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/MatchupToolMain.java @@ -21,10 +21,7 @@ package com.bc.fiduceo.matchup; import com.bc.fiduceo.log.FiduceoLogger; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; +import org.apache.commons.cli.*; public class MatchupToolMain { @@ -36,7 +33,7 @@ public static void main(String[] args) throws ParseException { return; } - final CommandLineParser parser = new PosixParser(); + final CommandLineParser parser = new DefaultParser(); final CommandLine commandLine = parser.parse(MatchupTool.getOptions(), args); if (commandLine.hasOption("h") || commandLine.hasOption("--help")) { matchupTool.printUsageTo(System.err); diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/BorderDistanceConditionPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/BorderDistanceConditionPlugin.java index d143c1ec6..ce0f792e9 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/BorderDistanceConditionPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/BorderDistanceConditionPlugin.java @@ -23,7 +23,7 @@ import static com.bc.fiduceo.util.JDomUtils.*; import static org.esa.snap.core.util.StringUtils.isNullOrEmpty; -import org.jdom.Element; +import org.jdom2.Element; import java.util.ArrayList; import java.util.HashSet; diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/Condition.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/Condition.java index fa15c3e94..e56a4bf51 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/Condition.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/Condition.java @@ -23,7 +23,7 @@ import com.bc.fiduceo.matchup.MatchupSet; -interface Condition { +public interface Condition { void apply(MatchupSet matchupSet, ConditionEngineContext context); } diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionEngine.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionEngine.java index 85b0d16ab..edae0679d 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionEngine.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionEngine.java @@ -26,7 +26,7 @@ import com.bc.fiduceo.matchup.MatchupSet; import com.bc.fiduceo.matchup.SampleSet; import com.bc.fiduceo.tool.ToolContext; -import org.jdom.Element; +import org.jdom2.Element; import java.util.ArrayList; import java.util.List; diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionFactory.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionFactory.java index 13dc7ae0a..6b949e838 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionFactory.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionFactory.java @@ -19,7 +19,7 @@ import com.bc.ceres.core.ServiceRegistry; import com.bc.ceres.core.ServiceRegistryManager; import org.esa.snap.core.util.ServiceLoader; -import org.jdom.Element; +import org.jdom2.Element; import java.util.HashMap; diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionPlugin.java index b2762ecf0..01484ecda 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/ConditionPlugin.java @@ -21,9 +21,9 @@ package com.bc.fiduceo.matchup.condition; -import org.jdom.Element; +import org.jdom2.Element; -interface ConditionPlugin { +public interface ConditionPlugin { Condition createCondition(Element element); diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/DistanceConditionPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/DistanceConditionPlugin.java index 193be6fb8..81d422fce 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/DistanceConditionPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/DistanceConditionPlugin.java @@ -23,7 +23,7 @@ import com.bc.fiduceo.matchup.SampleSet; import com.bc.fiduceo.util.JDomUtils; import org.esa.snap.core.util.StringUtils; -import org.jdom.Element; +import org.jdom2.Element; import java.util.ArrayList; import java.util.List; diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/OverlapRemoveConditionPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/OverlapRemoveConditionPlugin.java index 14297b99f..2ee94cbad 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/OverlapRemoveConditionPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/OverlapRemoveConditionPlugin.java @@ -23,7 +23,7 @@ import com.bc.fiduceo.matchup.SampleSet; import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Element; +import org.jdom2.Element; import java.util.stream.Stream; diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/PixelPositionConditionPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/PixelPositionConditionPlugin.java index ba9b9376d..cbd9f15bf 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/PixelPositionConditionPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/PixelPositionConditionPlugin.java @@ -2,7 +2,7 @@ import com.bc.fiduceo.util.JDomUtils; import org.esa.snap.core.util.StringUtils; -import org.jdom.Element; +import org.jdom2.Element; import java.util.stream.Stream; diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/TimeDeltaConditionPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/TimeDeltaConditionPlugin.java index 79eaf5704..11385a5d6 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/TimeDeltaConditionPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/TimeDeltaConditionPlugin.java @@ -25,7 +25,7 @@ import com.bc.fiduceo.matchup.SampleSet; import com.bc.fiduceo.util.JDomUtils; import org.esa.snap.core.util.StringUtils; -import org.jdom.Element; +import org.jdom2.Element; import java.util.ArrayList; import java.util.List; diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/UniqueSamplesConditionPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/UniqueSamplesConditionPlugin.java index d96f13156..2b864769c 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/UniqueSamplesConditionPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/condition/UniqueSamplesConditionPlugin.java @@ -22,7 +22,7 @@ import com.bc.fiduceo.util.JDomUtils; import org.esa.snap.core.util.StringUtils; -import org.jdom.Element; +import org.jdom2.Element; public class UniqueSamplesConditionPlugin implements ConditionPlugin { diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/plot/MapPlotTool.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/plot/MapPlotTool.java index b1e656734..3b18e973e 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/plot/MapPlotTool.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/plot/MapPlotTool.java @@ -68,7 +68,7 @@ public static void main(String[] args) throws IOException { samplingPoints.add(new SamplingPoint(lon, lat, time)); } - final String fileName = FileUtils.getFileNameFromPath(filePath); + final String fileName = FileUtils.getFilenameFromPath(filePath); final String pngFileName = FileUtils.exchangeExtension(fileName, ".png"); final String outputDir = new File(filePath).getParent(); //final File targetFile = new File(outputDir, args[4]); diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AngularCosineProportionScreeningPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AngularCosineProportionScreeningPlugin.java index 3c4571aa4..6d36d96b5 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AngularCosineProportionScreeningPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AngularCosineProportionScreeningPlugin.java @@ -22,7 +22,7 @@ import static com.bc.fiduceo.util.JDomUtils.*; -import org.jdom.Element; +import org.jdom2.Element; /* The XML template for this screening class looks like: diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AngularScreeningPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AngularScreeningPlugin.java index 2f8332c01..ce2ac8aa6 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AngularScreeningPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AngularScreeningPlugin.java @@ -20,8 +20,8 @@ package com.bc.fiduceo.matchup.screening; -import org.jdom.Attribute; -import org.jdom.Element; +import org.jdom2.Attribute; +import org.jdom2.Element; /* The XML template for this screening class looks like: diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AtsrAngularScreeningPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AtsrAngularScreeningPlugin.java index f6c370fe5..cf14b4aba 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AtsrAngularScreeningPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/AtsrAngularScreeningPlugin.java @@ -21,7 +21,7 @@ package com.bc.fiduceo.matchup.screening; import org.esa.snap.core.util.StringUtils; -import org.jdom.Element; +import org.jdom2.Element; public class AtsrAngularScreeningPlugin implements ScreeningPlugin { diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/BuehlerCloudScreeningPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/BuehlerCloudScreeningPlugin.java index 794f1ebae..eda6f9d93 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/BuehlerCloudScreeningPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/BuehlerCloudScreeningPlugin.java @@ -20,8 +20,8 @@ package com.bc.fiduceo.matchup.screening; -import org.jdom.Attribute; -import org.jdom.Element; +import org.jdom2.Attribute; +import org.jdom2.Element; public class BuehlerCloudScreeningPlugin implements ScreeningPlugin { diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/HIRS_LZADeltaScreeningPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/HIRS_LZADeltaScreeningPlugin.java index ea559b991..d190bf1e8 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/HIRS_LZADeltaScreeningPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/HIRS_LZADeltaScreeningPlugin.java @@ -21,7 +21,7 @@ package com.bc.fiduceo.matchup.screening; -import org.jdom.Element; +import org.jdom2.Element; public class HIRS_LZADeltaScreeningPlugin implements ScreeningPlugin { diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/PixelValueScreeningPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/PixelValueScreeningPlugin.java index 5e5e6bff5..cce491d94 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/PixelValueScreeningPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/PixelValueScreeningPlugin.java @@ -21,7 +21,7 @@ package com.bc.fiduceo.matchup.screening; -import org.jdom.Element; +import org.jdom2.Element; public class PixelValueScreeningPlugin implements ScreeningPlugin { diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningEngine.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningEngine.java index 6df0ce8cf..e5fe44d21 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningEngine.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningEngine.java @@ -25,7 +25,7 @@ import com.bc.fiduceo.matchup.MatchupSet; import com.bc.fiduceo.reader.Reader; import com.bc.fiduceo.tool.ToolContext; -import org.jdom.Element; +import org.jdom2.Element; import ucar.ma2.InvalidRangeException; import java.io.IOException; diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningFactory.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningFactory.java index 00006b576..f5f9aac45 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningFactory.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningFactory.java @@ -19,7 +19,7 @@ import com.bc.ceres.core.ServiceRegistry; import com.bc.ceres.core.ServiceRegistryManager; import org.esa.snap.core.util.ServiceLoader; -import org.jdom.Element; +import org.jdom2.Element; import java.util.HashMap; diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningPlugin.java index 815c10680..d8ad9bb10 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/ScreeningPlugin.java @@ -21,9 +21,9 @@ package com.bc.fiduceo.matchup.screening; -import org.jdom.Element; +import org.jdom2.Element; -interface ScreeningPlugin { +public interface ScreeningPlugin { Screening createScreening(Element element); diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/WindowValueScreeningPlugin.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/WindowValueScreeningPlugin.java index b42fc1fd2..d92e092f3 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/WindowValueScreeningPlugin.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/screening/WindowValueScreeningPlugin.java @@ -25,7 +25,7 @@ import com.bc.fiduceo.matchup.screening.WindowValueScreening.Evaluate; import com.bc.fiduceo.matchup.screening.WindowValueScreening.SecondaryConfiguration; import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Element; +import org.jdom2.Element; import java.util.ArrayList; import java.util.List; diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/strategy/InsituPolarOrbitingMatchupStrategy.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/strategy/InsituPolarOrbitingMatchupStrategy.java index 592bf2ceb..253ae3867 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/strategy/InsituPolarOrbitingMatchupStrategy.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/strategy/InsituPolarOrbitingMatchupStrategy.java @@ -346,6 +346,9 @@ private List getInsituSamplesPerSatellite(GeometryFactory geometryFa final List insituSamples = getInsituSamples(processingInterval, insituReader); for (final Sample insituSample : insituSamples) { final List candidatesByTime = getCandidatesByTime(secondaryObservations, new Date(insituSample.getTime()), timeDeltaInMillis); + if (candidatesByTime.size() == 0) { + continue; + } final Geometry point = geometryFactory.createPoint(insituSample.getLon(), insituSample.getLat()); final List candidatesByGeometry = getCandidatesByGeometry(candidatesByTime, point); diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/AbstractMmdWriter.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/AbstractMmdWriter.java index ec9278d80..adf1d5d45 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/AbstractMmdWriter.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/AbstractMmdWriter.java @@ -51,6 +51,11 @@ abstract class AbstractMmdWriter implements MmdWriter, Target { + static final String GLOBAL_ATTR_TITLE = "title"; + static final String GLOBAL_ATTR_INSTITUTION = "institution"; + static final String GLOBAL_ATTR_CONTACT = "contact"; + static final String GLOBAL_ATTR_LICENSE = "license"; + private final Logger logger; private final Map dataCacheMap; private final Map variableMap; @@ -291,9 +296,9 @@ void initializeNetcdfFile(Path mmdFile, UseCaseConfig useCaseConfig, List attributes = ioVariable.getAttributes(); for (Attribute attribute : attributes) { variable.addAttribute(attribute); @@ -366,10 +371,17 @@ private Variable getVariable(String variableName) { } private void createGlobalAttributes() { - addGlobalAttribute("title", "FIDUCEO multi-sensor match-up dataset (MMD)"); - addGlobalAttribute("institution", "Brockmann Consult GmbH"); - addGlobalAttribute("contact", "Tom Block (tom.block@brockmann-consult.de)"); - addGlobalAttribute("license", "This dataset is released for use under CC-BY licence and was developed in the EC FIDUCEO project \"Fidelity and Uncertainty in Climate Data Records from Earth Observations\". Grant Agreement: 638822."); + final Map ga = writerConfig.getGlobalAttributes(); + + final String title = ga.getOrDefault(GLOBAL_ATTR_TITLE, "SCEPS multi-sensor match-up dataset (MMD)"); + final String institution = ga.getOrDefault(GLOBAL_ATTR_INSTITUTION, "Brockmann Consult GmbH"); + final String contact = ga.getOrDefault(GLOBAL_ATTR_CONTACT, "Tom Block (tom.block@brockmann-consult.de)"); + final String licence = ga.getOrDefault(GLOBAL_ATTR_LICENSE, "ESA Data Policy: free and open access."); + + addGlobalAttribute(GLOBAL_ATTR_TITLE, title); + addGlobalAttribute(GLOBAL_ATTR_INSTITUTION, institution); + addGlobalAttribute(GLOBAL_ATTR_CONTACT, contact); + addGlobalAttribute(GLOBAL_ATTR_LICENSE, licence); addGlobalAttribute("creation_date", TimeUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss")); addGlobalAttribute("software_version", FiduceoConstants.VERSION); } diff --git a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/MmdWriterConfig.java b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/MmdWriterConfig.java index a78d6737e..b528c50ef 100644 --- a/matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/MmdWriterConfig.java +++ b/matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/MmdWriterConfig.java @@ -23,11 +23,11 @@ import com.bc.fiduceo.matchup.writer.MmdWriterFactory.NetcdfType; import org.esa.snap.core.util.StringUtils; -import org.jdom.Attribute; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.JDOMException; -import org.jdom.input.SAXBuilder; +import org.jdom2.Attribute; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.JDOMException; +import org.jdom2.input.SAXBuilder; import java.io.IOException; import java.io.InputStream; @@ -41,6 +41,7 @@ public class MmdWriterConfig { private static final String CACHE_SIZE_TAG = "cache-size"; private static final String NETCDF_FORMAT_TAG = "netcdf-format"; private static final String READER_CACHE_SIZE_TAG = "reader-cache-size"; + private static final String GLOBAL_ATTRIBUTES_TAG = "global-attributes"; private static final String VARIABLES_CONFIGURATION_TAG = "variables-configuration"; private static final String SENSOR_RENAME_TAG = "sensor-rename"; private static final String SEPARATOR = "separator"; @@ -52,6 +53,7 @@ public class MmdWriterConfig { private static final String SEPARATOR_ATTRIBUTE = "separator"; private static final String SENSOR_NAMES_ATTRIBUTE = "sensor-names"; private static final String NAMES_ATTRIBUTE = "names"; + private static final String VARIABLE_NAME_ATTRIBUTE = "variable-name"; private static final String VARIABLE_NAMES_ATTRIBUTE = "variable-names"; private static final String SOURCE_NAME_ATTRIBUTE = "source-name"; private static final String TARGET_NAME_ATTRIBUTE = "target-name"; @@ -61,12 +63,14 @@ public class MmdWriterConfig { private NetcdfType netcdfFormat; private VariablesConfiguration variablesConfiguration; private int readerCacheSize; + private Map globalAttributes; MmdWriterConfig() { cacheSize = 2048; netcdfFormat = NetcdfType.N4; variablesConfiguration = new VariablesConfiguration(); readerCacheSize = 6; + globalAttributes = new HashMap<>(); } private MmdWriterConfig(Document document) { @@ -96,6 +100,10 @@ public VariablesConfiguration getVariablesConfiguration() { return variablesConfiguration; } + public Map getGlobalAttributes() { + return Collections.unmodifiableMap(globalAttributes); + } + int getCacheSize() { return cacheSize; } @@ -151,6 +159,16 @@ private void init(Document document) { setReaderCacheSize(Integer.valueOf(readerCacheValue)); } + final Element globalAttributesElement = rootElement.getChild(GLOBAL_ATTRIBUTES_TAG); + if (globalAttributesElement != null) { + final List elements = globalAttributesElement.getChildren("attribute"); + for (Element element : elements) { + final String attName = element.getAttributeValue("name"); + final String attValue = element.getAttributeValue("value"); + globalAttributes.put(attName, attValue); + } + } + final Element variablesConfigurationElement = rootElement.getChild(VARIABLES_CONFIGURATION_TAG); if (variablesConfigurationElement != null) { addSensorRenames(variablesConfigurationElement); @@ -227,12 +245,20 @@ private void addAttributeRenames(Element sensorElement) { final String[] sensorNames = trim(sensorNamesStr.split(",")); final List renameAttributes = sensorElement.getChildren(RENAME_ATTRIBUTE_TAG); for (Element renameAttribute : renameAttributes) { - final Attribute varNamesAttr = renameAttribute.getAttribute(VARIABLE_NAMES_ATTRIBUTE); final String[] varNames; - if (varNamesAttr == null) { - varNames = new String[]{null}; + // a single rename + final Attribute varNameAttr = renameAttribute.getAttribute(VARIABLE_NAME_ATTRIBUTE); + if (varNameAttr != null) { + varNames = new String[]{varNameAttr.getValue()}; } else { - varNames = trim(varNamesAttr.getValue().split(",")); + // rename more than one variable + final Attribute varNamesAttr = renameAttribute.getAttribute(VARIABLE_NAMES_ATTRIBUTE); + + if (varNamesAttr == null) { + varNames = new String[]{null}; + } else { + varNames = trim(varNamesAttr.getValue().split(",")); + } } final String sourceName = getAttributeString(SOURCE_NAME_ATTRIBUTE, renameAttribute); final String targetName = getAttributeString(TARGET_NAME_ATTRIBUTE, renameAttribute); @@ -250,7 +276,7 @@ private void addWriteScaledConfig(Element sensorElement) { final List writeScaledElements = sensorElement.getChildren("writeScaled"); for (final Element scaledVarElement : writeScaledElements) { final String sourceName = getAttributeString(SOURCE_NAME_ATTRIBUTE, scaledVarElement); - for (final String sensorName: sensorNames) { + for (final String sensorName : sensorNames) { variablesConfiguration.addWriteScaled(sensorName, sourceName); } } diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest.java index 38db011a4..4af011b4a 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest.java @@ -34,7 +34,7 @@ public class MatchupToolIntegrationTest { private final String ls = System.lineSeparator(); - private final String expectedPrintUsage = "matchup-tool version 1.5.8" + ls + + private final String expectedPrintUsage = "matchup-tool version 1.6.2" + ls + ls + "usage: matchup-tool " + ls + "Valid options are:" + ls + diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_AMSUA_point.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_AMSUA_point.java new file mode 100644 index 000000000..cd8cdb34c --- /dev/null +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_AMSUA_point.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2017 Brockmann Consult GmbH + * This code was developed for the EC project "Fidelity and Uncertainty in + * Climate Data Records from Earth Observations (FIDUCEO)". + * Grant Agreement: 638822 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * A copy of the GNU General Public License should have been supplied along + * with this program; if not, see http://www.gnu.org/licenses/ + * + */ + +package com.bc.fiduceo.matchup; + +import com.bc.fiduceo.FiduceoConstants; +import com.bc.fiduceo.NCTestUtils; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.SatelliteObservation; +import com.bc.fiduceo.core.Sensor; +import com.bc.fiduceo.core.UseCaseConfig; +import com.bc.fiduceo.db.DbAndIOTestRunner; +import com.bc.fiduceo.util.NetCDFUtils; +import org.apache.commons.cli.ParseException; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.NetcdfFile; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +@RunWith(DbAndIOTestRunner.class) +public class MatchupToolIntegrationTest_AMSUA_point extends AbstractUsecaseIntegrationTest { + + @Test + public void testMatchup_AMSUA_location_extracts() throws IOException, ParseException, SQLException, InvalidRangeException { + final File mmdWriterConfig = new File(configDir, "mmd-writer-config.xml"); + if (!mmdWriterConfig.delete()) { + fail("unable to delete test file"); + } + TestUtil.writeMmdWriterConfig(configDir); + + final UseCaseConfig useCaseConfig = createUseCaseConfigBuilder() + .withLocationElement(150.1052, 20.8303) + .createConfig(); + final File useCaseConfigFile = storeUseCaseConfig(useCaseConfig, "usecase-amsua.xml"); + + insert_AMSUA(); + + final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-u", useCaseConfigFile.getName(), "-start", "2016-001", "-end", "2016-002"}; + MatchupToolMain.main(args); + + final File mmdFile = getMmdFilePath(useCaseConfig, "2016-001", "2016-002"); + assertTrue(mmdFile.isFile()); + + try (NetcdfFile mmd = NetcdfFile.open(mmdFile.getAbsolutePath())) { + final int matchupCount = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, mmd); + assertEquals(1, matchupCount); + + NCTestUtils.assert3DVariable("amsua-ma-l1b_SCENE_RADIANCE_01", 0, 0, 0, 10912, mmd); + NCTestUtils.assert3DVariable("amsua-ma-l1b_SCENE_RADIANCE_07", 1, 0, 0, 61606, mmd); + NCTestUtils.assert3DVariable("amsua-ma-l1b_SCENE_RADIANCE_14", 2, 0, 0, 74835, mmd); + NCTestUtils.assert3DVariable("amsua-ma-l1b_SURFACE_PROPERTIES", 0, 1, 0, 0, mmd); + NCTestUtils.assert3DVariable("amsua-ma-l1b_acquisition_time", 1, 1, 0, 1451693029, mmd); + NCTestUtils.assert3DVariable("amsua-ma-l1b_longitude", 2, 1, 0, 1494646, mmd); + NCTestUtils.assert3DVariable("amsua-ma-l1b_satellite_zenith_angle", 0, 2, 0, 4035, mmd); + } + } + + private void insert_AMSUA() throws IOException, SQLException { + final String sensorKey = "amsua-ma-l1b"; + final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "v8A", "2016", "01", "01", "AMSA_xxx_1B_M01_20160101234924Z_20160102013124Z_N_O_20160102003323Z.nat"}, true); + + final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "v8A"); + storage.insert(satelliteObservation); + } + + private MatchupToolTestUseCaseConfigBuilder createUseCaseConfigBuilder() { + final List sensorList = new ArrayList<>(); + final Sensor primary = new Sensor("amsua-ma-l1b"); + primary.setPrimary(true); + sensorList.add(primary); + + final List dimensions = new ArrayList<>(); + dimensions.add(new com.bc.fiduceo.core.Dimension("amsua-ma-l1b", 3, 3)); + + return (MatchupToolTestUseCaseConfigBuilder) new MatchupToolTestUseCaseConfigBuilder("amsua-ma-l1b") + .withSensors(sensorList) + .withOutputPath(new File(TestUtil.getTestDir().getPath(), "amsua-ma-l1b").getPath()) + .withDimensions(dimensions); + } +} diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_GBOV.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_GBOV.java new file mode 100644 index 000000000..d398c08d4 --- /dev/null +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_GBOV.java @@ -0,0 +1,171 @@ +package com.bc.fiduceo.matchup; + +import com.bc.fiduceo.FiduceoConstants; +import com.bc.fiduceo.NCTestUtils; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.SatelliteObservation; +import com.bc.fiduceo.core.Sensor; +import com.bc.fiduceo.core.UseCaseConfig; +import com.bc.fiduceo.db.DbAndIOTestRunner; +import com.bc.fiduceo.util.NetCDFUtils; +import org.apache.commons.cli.ParseException; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.NetcdfFile; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(DbAndIOTestRunner.class) +public class MatchupToolIntegrationTest_GBOV extends AbstractUsecaseIntegrationTest { + + @Test + public void testMatchup_gbov() throws IOException, SQLException, ParseException, InvalidRangeException { + final UseCaseConfig useCaseConfig = createUseCaseConfigBuilder("gbov") + .withTimeDeltaSeconds(900, null) + .withMaxPixelDistanceKm(50, null) + .createConfig(); + final File useCaseConfigFile = storeUseCaseConfig(useCaseConfig, "usecase-123.xml"); + + insert_gbov(); + insert_miras_CDF3TA_June(); + + final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-u", useCaseConfigFile.getName(), "-start", "2016-156", "-end", "2016-156"}; + MatchupToolMain.main(args); + + final File mmdFile = getMmdFilePath(useCaseConfig, "2016-156", "2016-156"); + assertTrue(mmdFile.isFile()); + + try (NetcdfFile mmd = NetcdfFile.open(mmdFile.getAbsolutePath())) { + + final int matchupCount = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, mmd); + assertEquals(30, matchupCount); + + NCTestUtils.assertStringVariable("gbov_site", null, 50, 0, "Barrow", mmd); + NCTestUtils.assertStringVariable("gbov_station", null, 50, 0, "Barrow", mmd); + NCTestUtils.assertStringVariable("gbov_IGBP_class", null, 50, 0, "Snow and Ice", mmd); + NCTestUtils.assert3DVariable("gbov_elevation", 0, 0, 0, 11.f, mmd); + NCTestUtils.assert3DVariable("gbov_Lat_IS", 0, 0, 0, 71.3231f, mmd); + NCTestUtils.assert3DVariable("gbov_Lon_IS", 0, 0, 0, -156.61052f, mmd); + + NCTestUtils.assert3DVariable("gbov_TIME_IS", 0, 0, 0, 1.46504898E9, mmd); + NCTestUtils.assert3DVariable("gbov_FIPAR_down", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_FIPAR_down_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_FIPAR_total", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_FIPAR_total_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_FIPAR_up", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_FIPAR_up_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_RM6_down_flag", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_RM6_up_flag", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_Clumping_Miller", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_Clumping_Miller_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_Clumping_Warren", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_Clumping_Warren_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAI_Miller_down", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAI_Miller_down_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAI_Miller_up", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAI_Miller_up_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAI_Warren_down", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAI_Warren_down_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAI_Warren_up", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAI_Warren_up_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAI_down", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAI_total_Miller", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAI_total_Warren", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAIe_Miller_down", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAIe_Miller_down_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAIe_Miller_up", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAIe_Miller_up_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAIe_Warren_down", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAIe_Warren_down_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAIe_Warren_up", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LAIe_Warren_up_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_PAI_Miller", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_PAI_Miller_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_PAI_Warren", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_PAI_Warren_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_PAIe_Miller", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_PAIe_Miller_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_PAIe_Warren", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_PAIe_Warren_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_clumping_Miller_down", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_clumping_Miller_down_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_clumping_Miller_up", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_clumping_Miller_up_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_clumping_Warren_down", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_clumping_Warren_down_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_clumping_Warren_up", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_clumping_Warren_up_err", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_RM7_down_flag", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_RM7_up_flag", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LSE", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LSE_STD", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LSR", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LSR_STD", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_QC_LSE", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_QC_LSR", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_LST", 0, 0, 12, 272.80047607421875f, mmd); + NCTestUtils.assert3DVariable("gbov_LST_STD", 0, 0, 12, 0.038089923560619354f, mmd); + NCTestUtils.assert3DVariable("gbov_QC_LST", 0, 0, 12, 0, mmd); + NCTestUtils.assert3DVariable("gbov_QC_SM_5", 0, 0, 12, -999.f, mmd); + NCTestUtils.assert3DVariable("gbov_SM_5", 0, 0, 12, -999.f, mmd); + + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Azimuth_Angle_175", 0, 0, 12, -22550.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_BT_3_125", 1, 0, 13, -32768.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_BT_4_175", 2, 0, 14, -996.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_BT_H_425", 0, 1, 15, -6808.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Days_175", 1, 1, 16, 5999.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Eta_325", 2, 1, 17, -3650.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Footprint_Axis2_125", 0, 2, 18, -21876.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Incidence_Angle_400", 1, 2, 0, -3615.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Nb_RFI_Flags_425", 2, 2, 1, 0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Nviews_575", 0, 0, 2, 20.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Pixel_BT_Standard_Deviation_H_075", 1, 0, 3, -32768.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Pixel_BT_Standard_Deviation_V_525", 2, 0, 4, -26642.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Pixel_Radiometric_Accuracy_H_025", 0, 1, 5, -32768.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_UTC_Microseconds_400", 1, 1, 6, 842685.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Xi_400", 2, 1, 7, -6608.0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_lat", 0, 2, 8, 71.7425537109375, mmd); + } + } + + private void insert_gbov() throws IOException, SQLException { + final String sensorKey = "gbov"; + final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", sensorKey, "v1", "2016", "06", "GBOV__Barrow__Barrow__20160601T000000Z__20160630T235900Z.csv"}, true); + + final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "v1"); + storage.insert(satelliteObservation); + } + + private void insert_miras_CDF3TA_June() throws IOException, SQLException { + final String sensorKey = "miras-smos-CDF3TA"; + final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "re07", "2016", "156", "SM_RE07_MIR_CDF3TA_20160604T000000_20160604T235959_330_001_7.tgz"}, true); + + final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "re07"); + storage.insert(satelliteObservation); + } + + private MatchupToolTestUseCaseConfigBuilder createUseCaseConfigBuilder(String sicSensor) { + final List sensorList = new ArrayList<>(); + final Sensor primary = new Sensor(sicSensor); + primary.setPrimary(true); + sensorList.add(primary); + sensorList.add(new Sensor("miras-smos-CDF3TA")); + + final List dimensions = new ArrayList<>(); + dimensions.add(new com.bc.fiduceo.core.Dimension(sicSensor, 1, 1)); + dimensions.add(new com.bc.fiduceo.core.Dimension("miras-smos-CDF3TA", 3, 3)); + + return (MatchupToolTestUseCaseConfigBuilder) new MatchupToolTestUseCaseConfigBuilder("mmd123") + .withSensors(sensorList) + .withOutputPath(new File(TestUtil.getTestDir().getPath(), "usecase-123").getPath()) + .withDimensions(dimensions); + } +} diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_MHS_point.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_MHS_point.java new file mode 100644 index 000000000..7af51ad93 --- /dev/null +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_MHS_point.java @@ -0,0 +1,86 @@ +package com.bc.fiduceo.matchup; + +import com.bc.fiduceo.FiduceoConstants; +import com.bc.fiduceo.NCTestUtils; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.SatelliteObservation; +import com.bc.fiduceo.core.Sensor; +import com.bc.fiduceo.core.UseCaseConfig; +import com.bc.fiduceo.db.DbAndIOTestRunner; +import com.bc.fiduceo.util.NetCDFUtils; +import org.apache.commons.cli.ParseException; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.NetcdfFile; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +@RunWith(DbAndIOTestRunner.class) +public class MatchupToolIntegrationTest_MHS_point extends AbstractUsecaseIntegrationTest { + + @Test + public void testMatchup_MHS_location_extracts() throws IOException, SQLException, ParseException, InvalidRangeException { + final File mmdWriterConfig = new File(configDir, "mmd-writer-config.xml"); + if (!mmdWriterConfig.delete()) { + fail("unable to delete test file"); + } + TestUtil.writeMmdWriterConfig(configDir); + + final UseCaseConfig useCaseConfig = createUseCaseConfigBuilder() + .withLocationElement(50, 60) + .createConfig(); + final File useCaseConfigFile = storeUseCaseConfig(useCaseConfig, "usecase-mhs.xml"); + + insert_MHS(); + + final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-u", useCaseConfigFile.getName(), "-start", "2025-232", "-end", "2025-233"}; + MatchupToolMain.main(args); + + final File mmdFile = getMmdFilePath(useCaseConfig, "2025-232", "2025-233"); + assertTrue(mmdFile.isFile()); + + try (NetcdfFile mmd = NetcdfFile.open(mmdFile.getAbsolutePath())) { + final int matchupCount = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, mmd); + assertEquals(1, matchupCount); + + NCTestUtils.assert3DVariable("mhs-mc-l1b_FOV_DATA_QUALITY", 0, 0, 0, 0, mmd); + NCTestUtils.assert3DVariable("mhs-mc-l1b_SCENE_RADIANCES_03", 1, 0, 0, 732588, mmd); + NCTestUtils.assert3DVariable("mhs-mc-l1b_SCENE_RADIANCES_05", 2, 0, 0, 868447, mmd); + NCTestUtils.assert3DVariable("mhs-mc-l1b_SURFACE_PROPERTIES", 0, 1, 0, 2, mmd); + NCTestUtils.assert3DVariable("mhs-mc-l1b_acquisition_time", 1, 1, 0, 1755669867, mmd); + NCTestUtils.assert3DVariable("mhs-mc-l1b_longitude", 2, 1, 0, 491101, mmd); + NCTestUtils.assert3DVariable("mhs-mc-l1b_satellite_zenith_angle", 0, 2, 0, 5302, mmd); + NCTestUtils.assert3DVariable("mhs-mc-l1b_TERRAIN_ELEVATION", 0, 2, 0, 0, mmd); + } + } + + private void insert_MHS() throws IOException, SQLException { + final String sensorKey = "mhs-mc-l1b"; + final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "v10", "2025", "08", "20", "MHSx_xxx_1B_M03_20250820060350Z_20250820074550Z_N_O_20250820074043Z.nat"}, true); + + final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "v10"); + storage.insert(satelliteObservation); + } + + private MatchupToolTestUseCaseConfigBuilder createUseCaseConfigBuilder() { + final List sensorList = new ArrayList<>(); + final Sensor primary = new Sensor("mhs-mc-l1b"); + primary.setPrimary(true); + sensorList.add(primary); + + final List dimensions = new ArrayList<>(); + dimensions.add(new com.bc.fiduceo.core.Dimension("mhs-mc-l1b", 3, 3)); + + return (MatchupToolTestUseCaseConfigBuilder) new MatchupToolTestUseCaseConfigBuilder("mhs-mc-l1b") + .withSensors(sensorList) + .withOutputPath(new File(TestUtil.getTestDir().getPath(), "mhs-mc-l1b").getPath()) + .withDimensions(dimensions); + } +} diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_SMAP_L2C_tao_sss.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_SMAP_L2C_tao_sss.java new file mode 100644 index 000000000..2e6a9afe4 --- /dev/null +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_SMAP_L2C_tao_sss.java @@ -0,0 +1,273 @@ +package com.bc.fiduceo.matchup; + +import com.bc.fiduceo.FiduceoConstants; +import com.bc.fiduceo.NCTestUtils; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.SatelliteObservation; +import com.bc.fiduceo.core.Sensor; +import com.bc.fiduceo.core.UseCaseConfig; +import com.bc.fiduceo.db.DbAndIOTestRunner; +import com.bc.fiduceo.util.NetCDFUtils; +import org.apache.commons.cli.ParseException; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.NetcdfFile; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.*; + +@RunWith(DbAndIOTestRunner.class) +public class MatchupToolIntegrationTest_SMAP_L2C_tao_sss extends AbstractUsecaseIntegrationTest { + + @Test + public void testMatchup_SMAP_L2C_sss__tao_sss() throws IOException, SQLException, ParseException, InvalidRangeException { + final UseCaseConfig useCaseConfig = createUseCaseConfigBuilder("tao-sss") + .withTimeDeltaSeconds(3600, null) + .withMaxPixelDistanceKm(14, null) + .createConfig(); + final File useCaseConfigFile = storeUseCaseConfig(useCaseConfig, "usecase-xxx.xml"); + + insert_tao_sss(); + insert_smap_sss_for(); + + final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-u", useCaseConfigFile.getName(), "-start", "2018-030", "-end", "2018-040"}; + MatchupToolMain.main(args); + + final File mmdFile = getMmdFilePath(useCaseConfig, "2018-030", "2018-040"); + assertTrue(mmdFile.isFile()); + + try (NetcdfFile mmd = NetcdfFile.open(mmdFile.getAbsolutePath())) { + + final int matchupCount = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, mmd); + assertEquals(2, matchupCount); + + NCTestUtils.assert3DVariable("smap-sss-for_time_for", 0, 0, 0, 5.710216511556213E8, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_cellat_for", 1, 0, 0, -0.12047244f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_cellon_for", 2, 0, 0, 249.86536f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_gland_for", 0, 1, 0, -1.4906111E-7f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_fland_for", 1, 1, 0, 8.9630685E-9f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_gice_est", 2, 1, 0, 0.0f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_surtep", 0, 2, 0, 296.19537f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_winspd", 1, 2, 0, 2.6554973f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_sss_ref", 2, 2, 0, 34.5771f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_tran", 0, 0, 1, 0.9898046f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_tbup", 1, 0, 1, 2.7570956f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_tbdw", 2, 0, 1, 2.7582273f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_windir", 0, 1, 1, 83.879845f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_rain", 1, 1, 1, 0.0f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_solar_flux", 2, 1, 1, 43.89048f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_temp_ant_for_V", 0, 2, 1, 381.10342f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_temp_ant_for_H", 1, 2, 1, 381.51367f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_zang_for", 2, 2, 1, 88.04912f, mmd); + + NCTestUtils.assert3DVariable("smap-sss-for_cellat_for", 1, 1, 0, 0.12653376f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_cellat_for", 1, 1, 1, 0.12653376f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_cellon_for", 1, 1, 0, 250.11238f, mmd); + NCTestUtils.assert3DVariable("smap-sss-for_cellon_for", 1, 1, 1, 250.11238f, mmd); + NCTestUtils.assert3DVariable("tao-sss_longitude", 0, 0, 0, -109.89f, mmd); + NCTestUtils.assert3DVariable("tao-sss_latitude", 0, 0, 0, 0.05f, mmd); + NCTestUtils.assert3DVariable("tao-sss_longitude", 0, 0, 1, -109.89f, mmd); + NCTestUtils.assert3DVariable("tao-sss_latitude", 0, 0, 1, 0.040273894f, mmd); + + NCTestUtils.assert3DVariable("tao-sss_time", 0, 0, 0, 1517706000, mmd); + NCTestUtils.assert3DVariable("tao-sss_time", 0, 0, 1, 1517709600, mmd); + NCTestUtils.assert3DVariable("tao-sss_SSS", 0, 0, 0, 34.754f, mmd); + NCTestUtils.assert3DVariable("tao-sss_SSS", 0, 0, 1, 34.751f, mmd); + NCTestUtils.assert3DVariable("tao-sss_SST", 0, 0, 0, 23.243f, mmd); + NCTestUtils.assert3DVariable("tao-sss_SST", 0, 0, 1, 23.272f, mmd); + NCTestUtils.assert3DVariable("tao-sss_AIRT", 0, 0, 0, -9.99f, mmd); + NCTestUtils.assert3DVariable("tao-sss_RH", 0, 0, 0, -9.99f, mmd); + + final String[] names = mmd.getVariables().stream() + .map(variable -> variable.getShortName()) + .collect(Collectors.toList()).toArray(new String[0]); + assertArrayEquals(new String[]{ + "smap-sss-for_time_for", + "smap-sss-for_cellat_for", + "smap-sss-for_cellon_for", + "smap-sss-for_gland_for", + "smap-sss-for_fland_for", + "smap-sss-for_gice_est", + "smap-sss-for_surtep", + "smap-sss-for_winspd", + "smap-sss-for_sss_ref", + "smap-sss-for_tran", + "smap-sss-for_tbup", + "smap-sss-for_tbdw", + "smap-sss-for_windir", + "smap-sss-for_rain", + "smap-sss-for_solar_flux", + "smap-sss-for_temp_ant_for_V", + "smap-sss-for_temp_ant_for_H", + "smap-sss-for_zang_for", + "smap-sss-for_alpha_for", + "smap-sss-for_eaa_for", + "smap-sss-for_eia_for", + "smap-sss-for_pra_for", + "smap-sss-for_sunglt_for", + "smap-sss-for_monglt_for", + "smap-sss-for_gallat_for", + "smap-sss-for_gallon_for", + "smap-sss-for_sun_beta_for", + "smap-sss-for_sun_alpha_for", + "smap-sss-for_ta_ant_filtered_for_V", + "smap-sss-for_ta_ant_filtered_for_H", + "smap-sss-for_ta_ant_filtered_for_S3", + "smap-sss-for_ta_ant_filtered_for_S4", + "smap-sss-for_ta_ant_for_V", + "smap-sss-for_ta_ant_for_H", + "smap-sss-for_ta_ant_for_S3", + "smap-sss-for_ta_ant_for_S4", + "smap-sss-for_dtemp_ant_for_V", + "smap-sss-for_dtemp_ant_for_H", + "smap-sss-for_ta_sun_dir_for_I", + "smap-sss-for_ta_sun_dir_for_Q", + "smap-sss-for_ta_sun_dir_for_S3", + "smap-sss-for_ta_sun_ref_for_I", + "smap-sss-for_ta_sun_ref_for_Q", + "smap-sss-for_ta_sun_ref_for_S3", + "smap-sss-for_ta_gal_dir_for_I", + "smap-sss-for_ta_gal_dir_for_Q", + "smap-sss-for_ta_gal_dir_for_S3", + "smap-sss-for_ta_gal_ref_for_I", + "smap-sss-for_ta_gal_ref_for_Q", + "smap-sss-for_ta_gal_ref_for_S3", + "smap-sss-for_ta_ant_calibrated_for_V", + "smap-sss-for_ta_ant_calibrated_for_H", + "smap-sss-for_ta_ant_calibrated_for_S3", + "smap-sss-for_ta_ant_calibrated_for_S4", + "smap-sss-for_ta_earth_for_V", + "smap-sss-for_ta_earth_for_H", + "smap-sss-for_ta_earth_for_S3", + "smap-sss-for_ta_earth_for_S4", + "smap-sss-for_tb_toi_for_V", + "smap-sss-for_tb_toi_for_H", + "smap-sss-for_tb_toi_for_S3", + "smap-sss-for_tb_toi_for_S4", + "smap-sss-for_tb_toa_for_V", + "smap-sss-for_tb_toa_for_H", + "smap-sss-for_tb_toa_for_S3", + "smap-sss-for_tb_toa_for_S4", + "smap-sss-for_tb_toa_lc_for_V", + "smap-sss-for_tb_toa_lc_for_H", + "smap-sss-for_tb_toa_lc_for_S3", + "smap-sss-for_tb_toa_lc_for_S4", + "smap-sss-for_dtb_land_correction_for_V", + "smap-sss-for_dtb_land_correction_for_H", + "smap-sss-for_dtb_sea_ice_correction_V", + "smap-sss-for_dtb_sea_ice_correction_H", + "smap-sss-for_tb_sur_for_V", + "smap-sss-for_tb_sur_for_H", + "smap-sss-for_tb_sur_for_S3", + "smap-sss-for_tb_sur_for_S4", + "smap-sss-for_tb_sur0_for_V", + "smap-sss-for_tb_sur0_for_H", + "smap-sss-for_tb_sur0_for_S3", + "smap-sss-for_tb_sur0_for_S4", + "smap-sss-for_tb_sur0_sic_for_V", + "smap-sss-for_tb_sur0_sic_for_H", + "smap-sss-for_tb_sur0_sic_for_S3", + "smap-sss-for_tb_sur0_sic_for_S4", + "smap-sss-for_sss_smap_for", + "smap-sss-for_sss_smap_40km_for", + "smap-sss-for_iqc_flag_for", + "smap-sss-for_anc_sea_ice_flag_ice1", + "smap-sss-for_anc_sea_ice_flag_ice2", + "smap-sss-for_anc_sea_ice_flag_ice3", + "smap-sss-for_sea_ice_zones", + "smap-sss-for_tb_consistency_for", + "smap-sss-for_ta_ant_exp_for_V", + "smap-sss-for_ta_ant_exp_for_H", + "smap-sss-for_ta_ant_exp_for_S3", + "smap-sss-for_ta_ant_exp_for_S4", + "smap-sss-for_tb_sur0_exp_for_V", + "smap-sss-for_tb_sur0_exp_for_H", + "smap-sss-for_tb_sur0_exp_for_S3", + "smap-sss-for_tb_sur0_exp_for_S4", + "smap-sss-for_pratot_exp_for", + "smap-sss-for_TEC_for", + "smap-sss-for_sss_smap_unc_for", + "smap-sss-for_sss_smap_40km_unc_for", + "smap-sss-for_sss_smap_unc_comp_ws-ran_for", + "smap-sss-for_sss_smap_unc_comp_nedt-v_for", + "smap-sss-for_sss_smap_unc_comp_nedt-h_for", + "smap-sss-for_sss_smap_unc_comp_sst_for", + "smap-sss-for_sss_smap_unc_comp_wdir_for", + "smap-sss-for_sss_smap_unc_comp_ref-gal_for", + "smap-sss-for_sss_smap_unc_comp_lnd-ctn_for", + "smap-sss-for_sss_smap_unc_comp_si-ctn_for", + "smap-sss-for_sss_smap_unc_comp_ws-sys_for", + "smap-sss-for_sss_smap_40km_unc_comp_ws-ran_for", + "smap-sss-for_sss_smap_40km_unc_comp_nedt-v_for", + "smap-sss-for_sss_smap_40km_unc_comp_nedt-h_for", + "smap-sss-for_sss_smap_40km_unc_comp_sst_for", + "smap-sss-for_sss_smap_40km_unc_comp_wdir_for", + "smap-sss-for_sss_smap_40km_unc_comp_ref-gal_for", + "smap-sss-for_sss_smap_40km_unc_comp_lnd-ctn_for", + "smap-sss-for_sss_smap_40km_unc_comp_si-ctn_for", + "smap-sss-for_sss_smap_40km_unc_comp_ws-sys_for", + "smap-sss-for_x", + "smap-sss-for_y", + "smap-sss-for_file_name", + "smap-sss-for_processing_version", + "smap-sss-for_acquisition_time", + "tao-sss_longitude", + "tao-sss_latitude", + "tao-sss_time", + "tao-sss_SSS", + "tao-sss_SST", + "tao-sss_AIRT", + "tao-sss_RH", + "tao-sss_WSPD", + "tao-sss_WDIR", + "tao-sss_BARO", + "tao-sss_RAIN", + "tao-sss_Q", + "tao-sss_M", + "tao-sss_x", + "tao-sss_y", + "tao-sss_file_name", + "tao-sss_processing_version", + "tao-sss_acquisition_time" + }, names); + } + } + + private void insert_tao_sss() throws IOException, SQLException { + final String sensorKey = "tao-sss"; + final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", sensorKey, "v1", "2018", "02", "TAO_T0N110W_DM234A-20170610_2018-02.txt"}, true); + final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "v1"); + storage.insert(satelliteObservation); + } + + private void insert_smap_sss_for() throws IOException, SQLException { + final String sensorKey = "smap-sss-for"; + final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "v05.0", "2018", "02", "04", "RSS_SMAP_SSS_L2C_r16080_20180204T004140_2018035_FNL_V05.0.nc"}, true); + final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "v05.0"); + storage.insert(satelliteObservation); + } + + private MatchupToolTestUseCaseConfigBuilder createUseCaseConfigBuilder(String insituName) { + final List sensorList = new ArrayList<>(); + final Sensor primary = new Sensor(insituName); + primary.setPrimary(true); + sensorList.add(primary); + sensorList.add(new Sensor("smap-sss-for")); + + final List dimensions = new ArrayList<>(); + dimensions.add(new com.bc.fiduceo.core.Dimension(insituName, 1, 1)); + dimensions.add(new com.bc.fiduceo.core.Dimension("smap-sss-for", 3, 3)); + + return (MatchupToolTestUseCaseConfigBuilder) new MatchupToolTestUseCaseConfigBuilder("mmdxxx") + .withSensors(sensorList) + .withOutputPath(new File(TestUtil.getTestDir().getPath(), "usecase-xxx").getPath()) + .withDimensions(dimensions); + } +} diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_SMOSL1_ndbc_sm.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_SMOSL1_ndbc_sm.java index 55e8f9484..386ef228e 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_SMOSL1_ndbc_sm.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_SMOSL1_ndbc_sm.java @@ -53,6 +53,7 @@ public void testMatchup_ndbc_sm() throws IOException, SQLException, ParseExcepti NCTestUtils.assert3DVariable("ndbc-sm-cs_PRES", 0, 0, 3, 1009.4f, mmd); NCTestUtils.assert3DVariable("ndbc-sm-cs_VIS", 0, 0, 4, 99.f, mmd); NCTestUtils.assert3DVariable("ndbc-sm-cs_WSPD", 0, 0, 5, 5.5f, mmd); + NCTestUtils.assert3DVariable("ndbc-sm-cs_WTMP", 0, 0, 3, 26.9f, mmd); NCTestUtils.assert3DVariable("ndbc-sm-cs_acquisition_time", 0, 0, 6, 1465042320L, mmd); NCTestUtils.assert3DVariable("ndbc-sm-cs_anemometer_height", 0, 0, 7, 10.f, mmd); NCTestUtils.assert3DVariable("ndbc-sm-cs_latitude", 0, 0, 8, 27.297f, mmd); diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_SMOSL1_tao_sss.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_SMOSL1_tao_sss.java new file mode 100644 index 000000000..2280e4ecc --- /dev/null +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_SMOSL1_tao_sss.java @@ -0,0 +1,120 @@ +package com.bc.fiduceo.matchup; + +import com.bc.fiduceo.FiduceoConstants; +import com.bc.fiduceo.NCTestUtils; +import com.bc.fiduceo.TestUtil; +import com.bc.fiduceo.core.SatelliteObservation; +import com.bc.fiduceo.core.Sensor; +import com.bc.fiduceo.core.UseCaseConfig; +import com.bc.fiduceo.db.DbAndIOTestRunner; +import com.bc.fiduceo.util.NetCDFUtils; +import org.apache.commons.cli.ParseException; +import org.junit.Test; +import org.junit.runner.RunWith; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.NetcdfFile; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(DbAndIOTestRunner.class) +public class MatchupToolIntegrationTest_SMOSL1_tao_sss extends AbstractUsecaseIntegrationTest { + + @Test + public void testMatchup_SMOS_TAO() throws IOException, SQLException, ParseException, InvalidRangeException { + final UseCaseConfig useCaseConfig = createUseCaseConfigBuilder() + .withTimeDeltaSeconds(7220, null) + .withMaxPixelDistanceKm(10, null) + .createConfig(); + final File useCaseConfigFile = storeUseCaseConfig(useCaseConfig, "usecase-45.xml"); + + insert_TAO_SSS(); + insert_miras_CDF3TA_June(); + + final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-u", useCaseConfigFile.getName(), "-start", "2016-156", "-end", "2016-156"}; + MatchupToolMain.main(args); + + + final File mmdFile = getMmdFilePath(useCaseConfig, "2016-156", "2016-156"); + assertTrue(mmdFile.isFile()); + try (NetcdfFile mmd = NetcdfFile.open(mmdFile.getAbsolutePath())) { + final int matchupCount = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, mmd); + assertEquals(4, matchupCount); + + NCTestUtils.assert3DVariable("tao-sss_AIRT", 0, 0, 0, 25.8f, mmd); + NCTestUtils.assert3DVariable("tao-sss_BARO", 0, 0, 1, -9.9f, mmd); + NCTestUtils.assertStringVariable("tao-sss_M", 8, 2, "DDDDDDDD", mmd); + NCTestUtils.assert3DVariable("tao-sss_Q", 0, 0, 3, 11111199, mmd); + NCTestUtils.assert3DVariable("tao-sss_RAIN", 0, 0, 0, -9.99f, mmd); + NCTestUtils.assert3DVariable("tao-sss_RH", 0, 0, 1, 87.33f, mmd); + NCTestUtils.assert3DVariable("tao-sss_SSS", 0, 0, 2, 35.49f, mmd); + NCTestUtils.assert3DVariable("tao-sss_SST", 0, 0, 3, 26.438f, mmd); + NCTestUtils.assert3DVariable("tao-sss_WDIR", 0, 0, 0, 257.f, mmd); + NCTestUtils.assert3DVariable("tao-sss_WSPD", 0, 0, 1, 4.7f, mmd); + NCTestUtils.assert3DVariable("tao-sss_acquisition_time", 0, 0, 2, 1465056000, mmd); + NCTestUtils.assertStringVariable("tao-sss_file_name", 128, 3, "TAO_T2S140W_DM167A-20160228_2016-06.txt", mmd); + NCTestUtils.assert3DVariable("tao-sss_latitude", 0, 0, 0, -2.04f, mmd); + NCTestUtils.assert3DVariable("tao-sss_longitude", 0, 0, 1, -139.99f, mmd); + NCTestUtils.assertStringVariable("tao-sss_processing_version", 30, 2, "v1", mmd); + NCTestUtils.assert3DVariable("tao-sss_time", 0, 0, 3, 1465059600, mmd); + NCTestUtils.assertVectorVariable("tao-sss_x", 0, 0, mmd); + NCTestUtils.assertVectorVariable("tao-sss_y", 1, 87, mmd); + + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Azimuth_Angle_175", 0, 0, 0, -32768, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_BT_3_125", 1, 0, 1, -32768, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_BT_4_175", 2, 0, 3, -32768, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_BT_H_425", 0, 1, 3, -19819, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Days_175", 1, 1, 0, -2147483647, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Eta_325", 2, 1, 1, -32768, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Footprint_Axis2_125", 0, 2, 2, -32768, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Incidence_Angle_400", 1, 2, 3, -3629, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Nb_RFI_Flags_425", 2, 2, 0, 0, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Nviews_575", 0, 0, 1, -32768, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Pixel_BT_Standard_Deviation_H_075", 1, 0, 2, -32768, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Pixel_BT_Standard_Deviation_V_525", 2, 0, 3, -32768, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Pixel_Radiometric_Accuracy_H_025", 0, 1, 0, -32768, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_UTC_Microseconds_400", 1, 1, 0, 289244, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_Xi_400", 2, 1, 2, 15486, mmd); + NCTestUtils.assert3DVariable("miras-smos-CDF3TA_lon", 0, 2, 3, -140.18731689453125, mmd); + } + } + + private void insert_TAO_SSS() throws IOException, SQLException { + final String sensorKey = "tao-sss"; + final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", sensorKey, "v1", "2016", "06", "TAO_T2S140W_DM167A-20160228_2016-06.txt"}, true); + + final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "v1"); + storage.insert(satelliteObservation); + } + + private void insert_miras_CDF3TA_June() throws IOException, SQLException { + final String sensorKey = "miras-smos-CDF3TA"; + final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "re07", "2016", "156", "SM_RE07_MIR_CDF3TA_20160604T000000_20160604T235959_330_001_7.tgz"}, true); + + final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "re07"); + storage.insert(satelliteObservation); + } + + private MatchupToolTestUseCaseConfigBuilder createUseCaseConfigBuilder() { + final List sensorList = new ArrayList<>(); + final Sensor primary = new Sensor("tao-sss"); + primary.setPrimary(true); + sensorList.add(primary); + sensorList.add(new Sensor("miras-smos-CDF3TA")); + + final List dimensions = new ArrayList<>(); + dimensions.add(new com.bc.fiduceo.core.Dimension("tao-sss", 1, 1)); + dimensions.add(new com.bc.fiduceo.core.Dimension("miras-smos-CDF3TA", 3, 3)); + + return (MatchupToolTestUseCaseConfigBuilder) new MatchupToolTestUseCaseConfigBuilder("mmd45") + .withSensors(sensorList) + .withOutputPath(new File(TestUtil.getTestDir().getPath(), "usecase-45").getPath()) + .withDimensions(dimensions); + } +} diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_Windsat_sic_cci.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_Windsat_sic_cci.java index 2f502aaca..4ec2333ac 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_Windsat_sic_cci.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_Windsat_sic_cci.java @@ -93,25 +93,17 @@ public void testMatchup() throws IOException, SQLException, ParseException, Inva @Test @Ignore public void test_oome_crash() throws IOException, SQLException, ParseException { - final UseCaseConfig useCaseConfig = createUseCaseConfigBuilder("DTUSIC1-sic-cci") + final UseCaseConfig useCaseConfig = createUseCaseConfigBuilder("ANTXXXI-sic-cci") .withTimeDeltaSeconds(86400, null) .withMaxPixelDistanceKm(14, null) .createConfig(); final File useCaseConfigFile = storeUseCaseConfig(useCaseConfig, "usecase-45.xml"); - insert_DTU_SIC_CCI("ASCAT-vs-AMSR2-vs-ERA5-vs-DTUSIC1-2018-S.text"); - insert_DTU_SIC_CCI("ASCAT-vs-AMSR2-vs-ERA5-vs-DTUSIC1-2018-N.text"); + insert_Polarstern(); - insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81462_20180930T055123_2018273_V08.0.nc"); - insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81465_20180930T105600_2018273_V08.0.nc"); - insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81466_20180930T123732_2018273_V08.0.nc"); - insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81467_20180930T141905_2018273_V08.0.nc"); - insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81468_20180930T160038_2018273_V08.0.nc"); - insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81469_20180930T174209_2018273_V08.0.nc"); - insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81470_20180930T192342_2018273_V08.0.nc"); - insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81471_20180930T210513_2018273_V08.0.nc"); + insert_windsat_MD("2016", "01", "15", "RSS_WindSat_TB_L1C_r67435_20160115T031230_2016015_V08.0.nc"); - final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-u", useCaseConfigFile.getName(), "-start", "2018-274", "-end", "2018-274"}; + final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-u", useCaseConfigFile.getName(), "-start", "2016-015", "-end", "2016-015"}; MatchupToolMain.main(args); } @@ -139,9 +131,17 @@ private void insert_DTU_SIC_CCI(String fileName) throws IOException, SQLExceptio storage.insert(satelliteObservation); } - private void insert_windsat_MD(String month, String day, String filename) throws IOException, SQLException { + private void insert_Polarstern() throws IOException, SQLException { + final String sensorKey = "ANTXXXI-sic-cci"; + final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "sic-cci", sensorKey, "v3", "ASCAT-vs-AMSR2-vs-ERA-vs-ANTXXXI_2_FROSN_SeaIceObservations_reformatted.txt"}, true); + + final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "v3"); + storage.insert(satelliteObservation); + } + + private void insert_windsat_MD(String year, String month, String day, String filename) throws IOException, SQLException { final String sensorKey = "windsat-coriolis"; - final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "v1.0", "2018", month, day, filename}, true); + final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "v1.0", year, month, day, filename}, true); final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "v1.0"); storage.insert(satelliteObservation); diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolTest.java index 11cb6e1e6..066b6f902 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolTest.java @@ -54,7 +54,7 @@ public void testPrintUsageTo() { matchupTool.printUsageTo(outputStream); - assertEquals("matchup-tool version 1.5.8" + ls + + assertEquals("matchup-tool version 1.6.2" + ls + ls + "usage: matchup-tool " + ls + "Valid options are:" + ls + diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolTestUseCaseConfigBuilder.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolTestUseCaseConfigBuilder.java index f233a4a81..64bfb093a 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolTestUseCaseConfigBuilder.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolTestUseCaseConfigBuilder.java @@ -23,7 +23,7 @@ import com.bc.fiduceo.matchup.condition.OverlapRemoveConditionPlugin; import com.bc.fiduceo.matchup.condition.TimeDeltaConditionPlugin; import org.esa.snap.core.util.StringUtils; -import org.jdom.Element; +import org.jdom2.Element; import static com.bc.fiduceo.core.UseCaseConfig.load; import static com.bc.fiduceo.util.JDomUtils.setNameAttribute; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/BorderDistanceConditionPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/BorderDistanceConditionPluginTest.java index 7b012bd75..6a479b765 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/BorderDistanceConditionPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/BorderDistanceConditionPluginTest.java @@ -25,8 +25,8 @@ import com.bc.fiduceo.TestUtil; import com.bc.fiduceo.matchup.SampleSet; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.*; import java.io.IOException; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/ConditionFactoryTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/ConditionFactoryTest.java index d8da47cc5..a0c721466 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/ConditionFactoryTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/ConditionFactoryTest.java @@ -21,8 +21,8 @@ package com.bc.fiduceo.matchup.condition; import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/DistanceConditionPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/DistanceConditionPluginTest.java index 14301c19c..4a4e467bd 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/DistanceConditionPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/DistanceConditionPluginTest.java @@ -27,8 +27,8 @@ import com.bc.fiduceo.TestUtil; import com.bc.fiduceo.matchup.SampleSet; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.*; import java.io.IOException; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/OverlapRemoveConditionPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/OverlapRemoveConditionPluginTest.java index cd36c168d..bc7f9695b 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/OverlapRemoveConditionPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/OverlapRemoveConditionPluginTest.java @@ -22,8 +22,8 @@ import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/PixelPositionConditionPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/PixelPositionConditionPluginTest.java index 31d96b230..4e4e8777f 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/PixelPositionConditionPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/PixelPositionConditionPluginTest.java @@ -1,8 +1,8 @@ package com.bc.fiduceo.matchup.condition; import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/TimeDeltaConditionPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/TimeDeltaConditionPluginTest.java index e77ff793f..d9007183b 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/TimeDeltaConditionPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/TimeDeltaConditionPluginTest.java @@ -24,8 +24,8 @@ import com.bc.fiduceo.matchup.MatchupSet; import com.bc.fiduceo.core.Sample; import com.bc.fiduceo.matchup.SampleSet; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/UniqueSamplesConditionPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/UniqueSamplesConditionPluginTest.java index e70f69132..19ee2ef6e 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/UniqueSamplesConditionPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/condition/UniqueSamplesConditionPluginTest.java @@ -21,8 +21,8 @@ package com.bc.fiduceo.matchup.condition; import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AngularCosineProportionScreeningPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AngularCosineProportionScreeningPluginTest.java index b3839c6b0..9d72b0578 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AngularCosineProportionScreeningPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AngularCosineProportionScreeningPluginTest.java @@ -21,7 +21,7 @@ package com.bc.fiduceo.matchup.screening; import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Element; +import org.jdom2.Element; import org.junit.Before; import org.junit.Test; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AngularScreeningPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AngularScreeningPluginTest.java index 12f59e014..80651d352 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AngularScreeningPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AngularScreeningPluginTest.java @@ -21,8 +21,8 @@ package com.bc.fiduceo.matchup.screening; import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Test; import java.io.IOException; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AtsrAngularScreeningPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AtsrAngularScreeningPluginTest.java index 67e24700a..f471e674f 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AtsrAngularScreeningPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/AtsrAngularScreeningPluginTest.java @@ -21,8 +21,8 @@ package com.bc.fiduceo.matchup.screening; import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/BuehlerCloudScreeningPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/BuehlerCloudScreeningPluginTest.java index 3cefa29e2..750bf42cd 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/BuehlerCloudScreeningPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/BuehlerCloudScreeningPluginTest.java @@ -21,8 +21,8 @@ package com.bc.fiduceo.matchup.screening; import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/HIRS_LZADeltaScreeningPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/HIRS_LZADeltaScreeningPluginTest.java index 70742b712..5339c764b 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/HIRS_LZADeltaScreeningPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/HIRS_LZADeltaScreeningPluginTest.java @@ -21,8 +21,8 @@ package com.bc.fiduceo.matchup.screening; import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/PixelValueScreeningPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/PixelValueScreeningPluginTest.java index 9c0fc1324..272138249 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/PixelValueScreeningPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/PixelValueScreeningPluginTest.java @@ -21,8 +21,8 @@ package com.bc.fiduceo.matchup.screening; import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/ScreeningFactoryTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/ScreeningFactoryTest.java index 7ca3d3563..38714a6c8 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/ScreeningFactoryTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/ScreeningFactoryTest.java @@ -21,8 +21,8 @@ package com.bc.fiduceo.matchup.screening; import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/WindowValueScreeningPluginTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/WindowValueScreeningPluginTest.java index 318d80314..8bc47efa2 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/WindowValueScreeningPluginTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/screening/WindowValueScreeningPluginTest.java @@ -25,8 +25,8 @@ import static org.junit.Assert.*; import com.bc.fiduceo.matchup.SampleSet; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.*; import java.io.IOException; diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/writer/MmdWriterConfigTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/writer/MmdWriterConfigTest.java index 289bb18fd..8607feae9 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/writer/MmdWriterConfigTest.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/writer/MmdWriterConfigTest.java @@ -21,6 +21,10 @@ package com.bc.fiduceo.matchup.writer; +import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_TITLE; +import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_INSTITUTION; +import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_CONTACT; +import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_LICENSE; import static com.bc.fiduceo.matchup.writer.MmdWriterFactory.NetcdfType.N3; import static com.bc.fiduceo.matchup.writer.MmdWriterFactory.NetcdfType.N4; import static org.junit.Assert.*; @@ -161,13 +165,56 @@ public void testLoad_emptyVariablesConfiguration() { assertNotNull(variablesConfiguration); } + @Test + public void testLoad_emptyGlobalAttributesConfiguration() { + //preparation + final String configXml = "" + + " " + + ""; + final ByteArrayInputStream inputStream = new ByteArrayInputStream(configXml.getBytes()); + + //execution + final MmdWriterConfig loadedConfig = MmdWriterConfig.load(inputStream); + final Map variablesConfiguration = loadedConfig.getGlobalAttributes(); + + //verification + assertNotNull(variablesConfiguration); + assertEquals(0, variablesConfiguration.size()); + } + + @Test + public void testLoad_fullGlobalAttributesConfiguration() { + //preparation + final String configXml = "" + + " " + + " " + + " " + + " " + + " " + + " " + + ""; + final ByteArrayInputStream inputStream = new ByteArrayInputStream(configXml.getBytes()); + + //execution + final MmdWriterConfig loadedConfig = MmdWriterConfig.load(inputStream); + final Map globAttr = loadedConfig.getGlobalAttributes(); + + //verification + assertNotNull(globAttr); + assertEquals(4, globAttr.size()); + assertEquals("Ein", globAttr.get(GLOBAL_ATTR_TITLE) ); + assertEquals("wunder", globAttr.get(GLOBAL_ATTR_INSTITUTION) ); + assertEquals("barer", globAttr.get(GLOBAL_ATTR_CONTACT) ); + assertEquals("Tag", globAttr.get(GLOBAL_ATTR_LICENSE) ); + } + @Test public void testLoad_AttributeRename() { final String configXml = "" + " " + " " + " " + - " " + + " " + " " + " " + " " + diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/writer/MmdWriter_IO_Test.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/writer/MmdWriter_IO_Test.java index b50a3abae..e86ebe712 100644 --- a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/writer/MmdWriter_IO_Test.java +++ b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/writer/MmdWriter_IO_Test.java @@ -45,6 +45,7 @@ import ucar.nc2.Variable; import ucar.nc2.iosp.netcdf3.N3iosp; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.PrintWriter; @@ -57,6 +58,10 @@ import java.util.Date; import java.util.List; +import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_TITLE; +import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_INSTITUTION; +import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_CONTACT; +import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_LICENSE; import static org.hamcrest.Matchers.equalToIgnoringWhiteSpace; import static org.junit.Assert.*; @@ -211,10 +216,10 @@ public void testCreate() throws IOException, InvalidRangeException { assertEquals(9, mmd.getGlobalAttributes().size()); - assertGlobalAttribute("title", "FIDUCEO multi-sensor match-up dataset (MMD)", mmd); + assertGlobalAttribute("title", "SCEPS multi-sensor match-up dataset (MMD)", mmd); assertGlobalAttribute("institution", "Brockmann Consult GmbH", mmd); assertGlobalAttribute("contact", "Tom Block (tom.block@brockmann-consult.de)", mmd); - assertGlobalAttribute("license", "This dataset is released for use under CC-BY licence and was developed in the EC FIDUCEO project \"Fidelity and Uncertainty in Climate Data Records from Earth Observations\". Grant Agreement: 638822.", mmd); + assertGlobalAttribute("license", "ESA Data Policy: free and open access.", mmd); assertGlobalDateAttribute("creation_date", TimeUtils.createNow(), mmd); assertGlobalAttribute("software_version", FiduceoConstants.VERSION, mmd); assertGlobalAttribute("sensor-names", "avhrr-n11,avhrr-n12", mmd); @@ -298,6 +303,67 @@ public void testCreate() throws IOException, InvalidRangeException { } } + @Test + public void testCreate_withUserDefinedGlobalAttributes() throws IOException, InvalidRangeException { + //preparation + final String configXml = "" + + " " + + " " + + " " + + " " + + " " + + " " + + ""; + final ByteArrayInputStream inputStream = new ByteArrayInputStream(configXml.getBytes()); + + final MmdWriterConfig loadedConfig = MmdWriterConfig.load(inputStream); + final MmdWriterNC3 mmdWriter = new MmdWriterNC3(loadedConfig); + + final Sensor primarySensor = new Sensor("avhrr-n11"); + primarySensor.setPrimary(true); + + final UseCaseConfig useCaseConfig = UseCaseConfigBuilder + .build("useCaseName") + .withDimensions(Arrays.asList( + new Dimension("avhrr-n11", 5, 7), + new Dimension("avhrr-n12", 3, 5))) + .withSensors(Arrays.asList( + primarySensor, + new Sensor("avhrr-n12"))) + .createConfig(); + + final Path mmdFile = Paths.get(testDir.toURI()).resolve("test_mmd.nc"); + +//execution + try { + mmdWriter.initializeNetcdfFile(mmdFile, useCaseConfig, new ArrayList(), 123); + } finally { + mmdWriter.close(); + } + + //verification + assertTrue(Files.isRegularFile(mmdFile)); + + NetcdfFile mmd = null; + try { + mmd = NetcdfFile.open(mmdFile.toString()); + + assertEquals(9, mmd.getGlobalAttributes().size()); + + assertGlobalAttribute("title", "Ein", mmd); + assertGlobalAttribute("institution", "wunder", mmd); + assertGlobalAttribute("contact", "schöner", mmd); + assertGlobalAttribute("license", "Tag", mmd); + assertGlobalDateAttribute("creation_date", TimeUtils.createNow(), mmd); + assertGlobalAttribute("software_version", FiduceoConstants.VERSION, mmd); + + } finally { + if (mmd != null) { + mmd.close(); + } + } + } + @Test public void testWrite_usecase02_AVHRR_NC3() throws IOException, InvalidRangeException { final MmdWriter mmdWriter = new MmdWriterNC3(writerConfig); diff --git a/pom.xml b/pom.xml index 8ebe4aa84..04b8ca76e 100644 --- a/pom.xml +++ b/pom.xml @@ -7,10 +7,10 @@ com.bc.fiduceo fiduceo-master pom - 1.5.8 + 1.6.2 - 1.5.8 + 1.6.2 @@ -27,37 +27,31 @@ org.esa.snap snap-core - 9.0.2 + 11.0.1 org.esa.snap snap-engine-utilities - 9.0.2 + 11.0.1 org.esa.snap snap-envisat-reader - 9.0.2 + 11.0.1 - org.esa.s3tbx - s3tbx-avhrr-reader - 9.0.3 + eu.esa.opt + opttbx-avhrr-reader + 11.0.1 - org.esa.s3tbx - s3tbx-sentinel3-reader - 9.0.3 - - - - - - + eu.esa.opt + opttbx-sentinel3-reader + 11.0.1 @@ -69,7 +63,7 @@ org.apache.commons commons-dbcp2 - 2.9.0 + 2.13.0 @@ -81,7 +75,7 @@ mysql mysql-connector-java - 8.0.18 + 8.0.33 @@ -98,7 +92,7 @@ com.h2database h2 - 2.1.210 + 2.3.232 @@ -117,17 +111,12 @@ netcdfAll 5.3.3 - - - - - - - - - - + + org.jdom + jdom2 + 2.0.6.1 + @@ -139,15 +128,23 @@ org.hamcrest - hamcrest-all - 1.3 + hamcrest + 3.0 test org.mockito mockito-core - 4.4.0 + + 4.11.0 + test + + + org.mockito + mockito-inline + + 4.11.0 test @@ -174,15 +171,23 @@ maven-jar-plugin 2.6 + + org.apache.maven.plugins + maven-resources-plugin + 3.3.1 + + UTF-8 + + maven-compiler-plugin - 3.10.1 + 3.11.0 - 1.8 - 1.8 + 11 + 11 true UTF-8 @@ -190,6 +195,10 @@ + + org.apache.maven.plugins + maven-resources-plugin + maven-assembly-plugin @@ -202,27 +211,19 @@ - - - - - - - - false maven-central Maven central repository - + https://repo.maven.apache.org/maven2/ snap-repo-public Public Maven Repository for SNAP - https://snap-build-server.tilaa.cloud/nexus/repository/snap-maven-public/ + https://nexus.snap-ci.ovh/repository/snap-maven-public/ true warn diff --git a/post-processing-tool/pom.xml b/post-processing-tool/pom.xml index d898b05c7..261879a4a 100644 --- a/post-processing-tool/pom.xml +++ b/post-processing-tool/pom.xml @@ -25,7 +25,7 @@ fiduceo-master com.bc.fiduceo - 1.5.8 + 1.6.2 4.0.0 @@ -64,9 +64,14 @@ mockito-core test + + org.mockito + mockito-inline + test + org.hamcrest - hamcrest-all + hamcrest test diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessing.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessing.java index 53ec68258..c62fb1cd3 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessing.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessing.java @@ -120,8 +120,10 @@ protected static ReaderCache createReaderCache(PostProcessingContext context) { protected static String getSourceFileName(Variable fileNameVar, int position, int filenameSize, final String fileNamePattern) throws IOException, InvalidRangeException { final String sourceFileName = NetCDFUtils.readString(fileNameVar, position, filenameSize); - if (!sourceFileName.matches(fileNamePattern)) { - throw new RuntimeException("The file name '" + sourceFileName + "' does not match the regular expression '" + fileNamePattern + "'"); + if (StringUtils.isNotNullAndNotEmpty(fileNamePattern)) { + if (!sourceFileName.matches(fileNamePattern)) { + throw new RuntimeException("The file name '" + sourceFileName + "' does not match the regular expression '" + fileNamePattern + "'"); + } } return sourceFileName; } @@ -129,9 +131,8 @@ protected static String getSourceFileName(Variable fileNameVar, int position, in protected String getMatchupDimensionName() { String matchupDimensionName = context.getProcessingConfig().getMatchupDimensionName(); if (StringUtils.isNullOrEmpty(matchupDimensionName)) { - matchupDimensionName = FiduceoConstants.MATCHUP_COUNT; + return FiduceoConstants.MATCHUP_COUNT; } - return matchupDimensionName; } } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingConfig.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingConfig.java index 56b08caf2..926d0ea8f 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingConfig.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingConfig.java @@ -20,12 +20,12 @@ package com.bc.fiduceo.post; import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.JDOMException; -import org.jdom.input.SAXBuilder; -import org.jdom.output.Format; -import org.jdom.output.XMLOutputter; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.JDOMException; +import org.jdom2.input.SAXBuilder; +import org.jdom2.output.Format; +import org.jdom2.output.XMLOutputter; import java.io.IOException; import java.io.InputStream; @@ -119,7 +119,7 @@ private void init() { final Element processingsElem = JDomUtils.getMandatoryChild(rootElement, TAG_NAME_POST_PROCESSINGS); postProcessingElements = processingsElem.getChildren(); - if (postProcessingElements.size() == 0) { + if (postProcessingElements.isEmpty()) { throw new RuntimeException("Empty list of post processings."); } } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingContext.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingContext.java index b4f919adc..79f8b71ce 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingContext.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingContext.java @@ -26,6 +26,7 @@ public final class PostProcessingContext extends ToolContext { private PostProcessingConfig processingConfig; private Path mmdInputDirectory; + private Path configDirectory; public PostProcessingConfig getProcessingConfig() { return processingConfig; @@ -42,4 +43,12 @@ public Path getMmdInputDirectory() { public void setMmdInputDirectory(Path mmdInputDirectory) { this.mmdInputDirectory = mmdInputDirectory; } + + public void setConfigDirectory(Path configDirectory) { + this.configDirectory = configDirectory; + } + + public Path getConfigDirectory() { + return configDirectory; + } } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingFactory.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingFactory.java index bceaa8726..b4643c3bb 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingFactory.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingFactory.java @@ -22,7 +22,7 @@ import com.bc.ceres.core.ServiceRegistry; import com.bc.ceres.core.ServiceRegistryManager; import org.esa.snap.core.util.ServiceLoader; -import org.jdom.Element; +import org.jdom2.Element; import java.util.Collections; import java.util.HashMap; @@ -51,14 +51,17 @@ public static PostProcessingFactory get() { return postProcessingFactory; } - public PostProcessing getPostProcessing(Element element) { + public PostProcessing getPostProcessing(Element element, PostProcessingContext context) { final String name = element.getName(); final PostProcessingPlugin plugin = postProcessingPluginMap.get(name); if (plugin == null) { throw new IllegalArgumentException("PostProcessing for name '" + name + "' not available."); } - return plugin.createPostProcessing(element); + + final PostProcessing postProcessing = plugin.createPostProcessing(element, context); + postProcessing.setContext(context); + return postProcessing; } // package access for testing only se 2016-11-28 diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingPlugin.java index abaff5ff7..de46efb3f 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingPlugin.java @@ -19,11 +19,11 @@ package com.bc.fiduceo.post; -import org.jdom.Element; +import org.jdom2.Element; public interface PostProcessingPlugin { - PostProcessing createPostProcessing(Element element); + PostProcessing createPostProcessing(Element element, PostProcessingContext context); String getPostProcessingName(); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingTool.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingTool.java index 520b4f298..f5c11c2ed 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingTool.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingTool.java @@ -34,7 +34,7 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.esa.snap.core.util.StringUtils; -import org.jdom.Element; +import org.jdom2.Element; import ucar.ma2.InvalidRangeException; import ucar.nc2.*; import ucar.nc2.constants.DataFormatType; @@ -71,6 +71,7 @@ static PostProcessingContext initializeContext(CommandLine commandLine) throws I final String configValue = commandLine.getOptionValue("config", "./config"); final Path configDirectory = Paths.get(configValue); + context.setConfigDirectory(configDirectory); final SystemConfig systemConfig = SystemConfig.loadFrom(configDirectory.toFile()); context.setSystemConfig(systemConfig); @@ -193,8 +194,7 @@ private void computeFiles(List mmdFiles) throws Exception { final List processings = new ArrayList<>(); final PostProcessingFactory factory = PostProcessingFactory.get(); for (Element processing : processingConfig.getPostProcessingElements()) { - final PostProcessing postProcessing = factory.getPostProcessing(processing); - postProcessing.setContext(context); + final PostProcessing postProcessing = factory.getPostProcessing(processing, context); processings.add(postProcessing); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingToolMain.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingToolMain.java index 9cdd7d77a..26aafff68 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingToolMain.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingToolMain.java @@ -23,15 +23,12 @@ import com.bc.fiduceo.log.FiduceoLogger; import com.bc.fiduceo.tool.ShutdownHook; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; +import org.apache.commons.cli.*; public class PostProcessingToolMain { public static void main(String[] args) { - final CommandLineParser parser = new PosixParser(); + final CommandLineParser parser = new DefaultParser(); final CommandLine commandLine; try { commandLine = parser.parse(PostProcessingTool.getOptions(), args); diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/ElevationToSolZenAnglePlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/ElevationToSolZenAnglePlugin.java index c9cf4d1a3..27e1444f1 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/ElevationToSolZenAnglePlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/ElevationToSolZenAnglePlugin.java @@ -2,10 +2,11 @@ import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; import com.bc.fiduceo.util.JDomUtils; import org.esa.snap.core.util.StringUtils; -import org.jdom.Element; +import org.jdom2.Element; import java.util.List; @@ -22,7 +23,7 @@ public class ElevationToSolZenAnglePlugin implements PostProcessingPlugin { private static final String ROOT_TAG_NAME = "elevation-to-solzen-angle"; @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { final ElevationToSolZenAngle.Configuration configuration = createConfiguration(element); return new ElevationToSolZenAngle(configuration); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/airs/AddAirsSpectrumPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/airs/AddAirsSpectrumPlugin.java index 3ad3b3701..1b8412ea3 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/airs/AddAirsSpectrumPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/airs/AddAirsSpectrumPlugin.java @@ -22,8 +22,9 @@ import static com.bc.fiduceo.util.JDomUtils.getMandatoryChildMandatoryTextTrim; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; -import org.jdom.Element; +import org.jdom2.Element; public class AddAirsSpectrumPlugin implements PostProcessingPlugin { @@ -42,7 +43,7 @@ public class AddAirsSpectrumPlugin implements PostProcessingPlugin { // (GeoTrack=135, Channel=2378); @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { if (!getPostProcessingName().equals(element.getName())) { throw new RuntimeException("Illegal XML Element. Tagname '" + getPostProcessingName() + "' expected."); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/amsr/AddAmsr2ScanDataQualityPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/amsr/AddAmsr2ScanDataQualityPlugin.java index 612760f17..352d2b221 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/amsr/AddAmsr2ScanDataQualityPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/amsr/AddAmsr2ScanDataQualityPlugin.java @@ -21,15 +21,16 @@ package com.bc.fiduceo.post.plugin.amsr; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Attribute; -import org.jdom.Element; +import org.jdom2.Attribute; +import org.jdom2.Element; public class AddAmsr2ScanDataQualityPlugin implements PostProcessingPlugin { @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { final AddAmsr2ScanDataQuality.Configuration configuration = createConfiguration(element); final AddAmsr2ScanDataQuality postProcessing = new AddAmsr2ScanDataQuality(); diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/amsr/AddAmsrSolarAnglesPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/amsr/AddAmsrSolarAnglesPlugin.java index 7e63195e8..170510074 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/amsr/AddAmsrSolarAnglesPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/amsr/AddAmsrSolarAnglesPlugin.java @@ -21,10 +21,11 @@ package com.bc.fiduceo.post.plugin.amsr; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Attribute; -import org.jdom.Element; +import org.jdom2.Attribute; +import org.jdom2.Element; /* The XML template for this post processing class looks like: @@ -41,7 +42,7 @@ public class AddAmsrSolarAnglesPlugin implements PostProcessingPlugin { @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { final AddAmsrSolarAngles.Configuration configuration = createConfiguration(element); final AddAmsrSolarAngles solarAnglesPostProcessing = new AddAmsrSolarAngles(); diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffs.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffs.java index 69ed3f7d0..31617a661 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffs.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffs.java @@ -7,7 +7,7 @@ import com.bc.fiduceo.reader.fiduceo_fcdr.AVHRR_FCDR_Reader; import com.bc.fiduceo.util.JDomUtils; import com.bc.fiduceo.util.NetCDFUtils; -import org.jdom.Element; +import org.jdom2.Element; import ucar.ma2.*; import ucar.nc2.Dimension; import ucar.nc2.NetcdfFile; diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsPlugin.java index 30c67d0cc..7203a1a29 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsPlugin.java @@ -1,13 +1,14 @@ package com.bc.fiduceo.post.plugin.avhrr_fcdr; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; -import org.jdom.Element; +import org.jdom2.Element; public class AddAvhrrCorrCoeffsPlugin implements PostProcessingPlugin { @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { final AddAvhrrCorrCoeffs.Configuration configuration = AddAvhrrCorrCoeffs.createConfiguration(element); return new AddAvhrrCorrCoeffs(configuration); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/caliop/flag/CALIOP_L2_VFM_FLAGS_PPPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/caliop/flag/CALIOP_L2_VFM_FLAGS_PPPlugin.java index 9e216f166..5d16197bd 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/caliop/flag/CALIOP_L2_VFM_FLAGS_PPPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/caliop/flag/CALIOP_L2_VFM_FLAGS_PPPlugin.java @@ -23,8 +23,9 @@ import static com.bc.fiduceo.util.JDomUtils.*; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; -import org.jdom.Element; +import org.jdom2.Element; public class CALIOP_L2_VFM_FLAGS_PPPlugin implements PostProcessingPlugin { @@ -37,7 +38,7 @@ public class CALIOP_L2_VFM_FLAGS_PPPlugin implements PostProcessingPlugin { @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { if (!getPostProcessingName().equals(element.getName())) { throw new RuntimeException("Illegal XML Element. Tagname '" + getPostProcessingName() + "' expected."); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/caliop/sst_wp100/CALIOP_SST_WP100_CLay_PPPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/caliop/sst_wp100/CALIOP_SST_WP100_CLay_PPPlugin.java index 9a75b6c15..b7e2f9aa4 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/caliop/sst_wp100/CALIOP_SST_WP100_CLay_PPPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/caliop/sst_wp100/CALIOP_SST_WP100_CLay_PPPlugin.java @@ -23,8 +23,9 @@ import static com.bc.fiduceo.util.JDomUtils.getMandatoryChildMandatoryTextTrim; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; -import org.jdom.Element; +import org.jdom2.Element; public class CALIOP_SST_WP100_CLay_PPPlugin implements PostProcessingPlugin { @@ -36,7 +37,7 @@ public class CALIOP_SST_WP100_CLay_PPPlugin implements PostProcessingPlugin { @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { if (!getPostProcessingName().equals(element.getName())) { throw new RuntimeException("Illegal XML Element. Tagname '" + getPostProcessingName() + "' expected."); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/ArrayUtils.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/ArrayUtils.java new file mode 100644 index 000000000..b4c695d3e --- /dev/null +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/ArrayUtils.java @@ -0,0 +1,62 @@ +package com.bc.fiduceo.post.plugin.era5; + +import ucar.ma2.Array; + +class ArrayUtils { + + static Array mergeAlongX(Array left, Array right) { + final int[] leftShape = left.getShape(); + final int[] rightShape = right.getShape(); + final int rank = left.getRank(); + + final Object leftStorage = left.getStorage(); + final Object rightStorage = right.getStorage(); + + Array targetArray; + if (rank == 3) { + final int width = leftShape[2] + rightShape[2]; + final int[] targetShape = new int[]{leftShape[0], leftShape[1], width}; + targetArray = Array.factory(left.getDataType(), targetShape); + final Object targetStorage = targetArray.getStorage(); + + int srcOffsetLeft = 0; + int srcOffsetRight = 0; + int destOffset = 0; + for (int z = 0; z < leftShape[0]; z++) { + for (int y = 0; y < leftShape[1]; y++) { + System.arraycopy(leftStorage, srcOffsetLeft, targetStorage, destOffset, leftShape[2]); + destOffset += leftShape[2]; + srcOffsetLeft += leftShape[2]; + System.arraycopy(rightStorage, srcOffsetRight, targetStorage, destOffset, rightShape[2]); + srcOffsetRight += rightShape[2]; + destOffset += rightShape[2]; + } + } + } else if (rank == 4) { + final int width = leftShape[3] + rightShape[3]; + final int[] targetShape = new int[]{leftShape[0], leftShape[1], leftShape[2], width}; + targetArray = Array.factory(left.getDataType(), targetShape); + final Object targetStorage = targetArray.getStorage(); + + int srcOffsetLeft = 0; + int srcOffsetRight = 0; + int destOffset = 0; + for (int layer = 0; layer < leftShape[0]; layer++) { + for (int z = 0; z < leftShape[1]; z++) { + for (int y = 0; y < leftShape[2]; y++) { + System.arraycopy(leftStorage, srcOffsetLeft, targetStorage, destOffset, leftShape[3]); + destOffset += leftShape[3]; + srcOffsetLeft += leftShape[3]; + System.arraycopy(rightStorage, srcOffsetRight, targetStorage, destOffset, rightShape[3]); + srcOffsetRight += rightShape[3]; + destOffset += rightShape[3]; + } + } + } + + } else { + throw new IllegalArgumentException("invalid input data rank"); + } + return targetArray; + } +} diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/BilinearInterpolator.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/BilinearInterpolator.java index faef36637..1fbafe680 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/BilinearInterpolator.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/BilinearInterpolator.java @@ -6,12 +6,16 @@ class BilinearInterpolator { private final double b; private final int xMin; private final int yMin; + private int relXMin; + private int relYMin; BilinearInterpolator(double a, double b, int xMin, int yMin) { this.a = a; this.b = b; this.xMin = xMin; this.yMin = yMin; + relXMin = Integer.MIN_VALUE; + relYMin = Integer.MIN_VALUE; } double interpolate(float c00, float c10, float c01, float c11) { @@ -28,9 +32,29 @@ int getYMin() { return yMin; } - double getA() {return a;} + public int getRelXMin() { + return relXMin; + } + + public void setRelXMin(int relXMin) { + this.relXMin = relXMin; + } + + public int getRelYMin() { + return relYMin; + } - double getB() {return b;} + public void setRelYMin(int relYMin) { + this.relYMin = relYMin; + } + + double getA() { + return a; + } + + double getB() { + return b; + } private static double lerp(double c0, double c1, double t) { return c0 + (c1 - c0) * t; diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Configuration.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Configuration.java index 13e67bd4c..8fbc1345b 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Configuration.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Configuration.java @@ -4,6 +4,7 @@ class Configuration { private String nwpAuxDir; private String era5Collection; + private boolean translateVariableNameToFileAccessName = true; private SatelliteFieldsConfiguration satelliteFields; private MatchupFieldsConfiguration matchupFields; @@ -39,4 +40,12 @@ MatchupFieldsConfiguration getMatchupFields() { void setMatchupFields(MatchupFieldsConfiguration matchupFields) { this.matchupFields = matchupFields; } + + public boolean isTranslateVariableNameToFileAccessName() { + return translateVariableNameToFileAccessName; + } + + public void setTranslateVariableNameToFileAccessName(boolean translateVariableNameToFileAccessName) { + this.translateVariableNameToFileAccessName = translateVariableNameToFileAccessName; + } } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5Archive.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5Archive.java index c5ea93918..0acd0f1a5 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5Archive.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5Archive.java @@ -1,6 +1,7 @@ package com.bc.fiduceo.post.plugin.era5; import com.bc.fiduceo.util.TimeUtils; +import org.apache.commons.lang3.StringUtils; import java.io.File; import java.text.DecimalFormat; @@ -10,11 +11,11 @@ class Era5Archive { private static final DecimalFormat twoDigitsFormat = new DecimalFormat("00"); - private final String rootPath; private final Era5Collection collection; + private final Configuration config; - Era5Archive(String rootPath, Era5Collection collection) { - this.rootPath = rootPath; + Era5Archive(Configuration config, Era5Collection collection) { + this.config = config; this.collection = collection; } @@ -86,11 +87,13 @@ String get(String variableType, int timeStamp) { final Calendar utcCalendar = TimeUtils.getUTCCalendar(); utcCalendar.setTimeInMillis(timeStamp * 1000L); - final int cutPoint = variableType.lastIndexOf("_"); + final int cutPoint = StringUtils.ordinalIndexOf(variableType, "_", 2); final String collection = variableType.substring(0, cutPoint); String variable = variableType.substring(cutPoint + 1); - variable = mapVariable(variable); + if (config.isTranslateVariableNameToFileAccessName()) { + variable = mapVariable(variable); + } adjustCalendarForForecast(utcCalendar, collection); @@ -105,9 +108,13 @@ String get(String variableType, int timeStamp) { final int day = utcCalendar.get(Calendar.DAY_OF_MONTH); final String dayString = twoDigitsFormat.format(day); - return rootPath + File.separator + collection + File.separator + - year + File.separator + monthString + File.separator + dayString + File.separator + - fileName; + final String rootPath = config.getNWPAuxDir(); + return rootPath + File.separator + + collection + File.separator + + year + File.separator + + monthString + File.separator + + dayString + File.separator + + fileName; } // @todo 1 tb/tb make static and add test 2020-12-11 diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java index 518225cb8..0d7782a5e 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java @@ -9,12 +9,12 @@ import ucar.nc2.NetcdfFile; import ucar.nc2.NetcdfFileWriter; -import java.awt.*; import java.io.IOException; class Era5PostProcessing extends PostProcessing { private static final double EPS = 0.00000001; + public static final int DATA_ARRAY_WIDTH = 1440; private final Configuration configuration; @@ -37,34 +37,35 @@ static int getEra5LatMin(float latMax) { static int getEra5LonMin(float lonMin) { final double normLonMin = (lonMin + 360.0) % 360.0; - final double scaledLonMin = Math.floor(normLonMin * 4) / 4; - return (int) (scaledLonMin * 4); + final double xIndex = Math.floor(normLonMin * 4); // lon index + return (int) xIndex; } // package access for testing only tb 2020-11-20 static InterpolationContext getInterpolationContext(Array lonArray, Array latArray) { final int[] shape = lonArray.getShape(); if (shape.length == 2) { - return createInterpolationContext_2D(lonArray, latArray, shape); + return createInterpolationContext_2D(lonArray, latArray); } else if (shape.length == 0) { - return createInterpolationContext_1D(lonArray, latArray); + return createInterpolationContext_0D(lonArray, latArray); } throw new IllegalStateException("Unsupported dimensionality of geolocation data"); } - private static InterpolationContext createInterpolationContext_2D(Array lonArray, Array latArray, int[] shape) { + private static InterpolationContext createInterpolationContext_2D(Array lonArray, Array latArray) { + final int[] shape = lonArray.getShape(); final InterpolationContext context = new InterpolationContext(shape[1], shape[0]); final Index lonIdx = lonArray.getIndex(); final Index latIdx = latArray.getIndex(); - int xMin = Integer.MAX_VALUE; - int xMax = Integer.MIN_VALUE; - int yMin = Integer.MAX_VALUE; - int yMax = Integer.MIN_VALUE; for (int y = 0; y < shape[0]; y++) { for (int x = 0; x < shape[1]; x++) { - lonIdx.set(y, x); + if (x >= DATA_ARRAY_WIDTH) { + lonIdx.set(y, x - DATA_ARRAY_WIDTH); + } else { + lonIdx.set(y, x); + } latIdx.set(y, x); final float lon = lonArray.getFloat(lonIdx); @@ -81,27 +82,12 @@ private static InterpolationContext createInterpolationContext_2D(Array lonArray // + store to context at (x, y) final int era5_X_min = getEra5LonMin(lon); final int era5_Y_min = getEra5LatMin(lat); - if (era5_X_min < xMin) { - xMin = era5_X_min; - } - if (era5_X_min > xMax) { - xMax = era5_X_min; - } - if (era5_Y_min < yMin) { - yMin = era5_Y_min; - } - if (era5_Y_min > yMax) { - yMax = era5_Y_min; - } final BilinearInterpolator interpolator = createInterpolator(lon, lat, era5_X_min, era5_Y_min); context.set(x, y, interpolator); } - - // add 2 to width and height to have always 4 points for the interpolation tb 2020-11-30 - final Rectangle era5Rect = new Rectangle(xMin, yMin, xMax - xMin + 2, yMax - yMin + 2); - context.setEra5Region(era5Rect); } + return context; } @@ -119,7 +105,7 @@ static BilinearInterpolator createInterpolator(float lon, float lat, int era5_X_ return new BilinearInterpolator(lonDelta, latDelta, era5_X_min, era5_Y_min); } - private static InterpolationContext createInterpolationContext_1D(Array lonArray, Array latArray) { + private static InterpolationContext createInterpolationContext_0D(Array lonArray, Array latArray) { final InterpolationContext context = new InterpolationContext(1, 1); final float lon = lonArray.getFloat(0); @@ -131,9 +117,6 @@ private static InterpolationContext createInterpolationContext_1D(Array lonArray final BilinearInterpolator interpolator = createInterpolator(lon, lat, era5_X_min, era5_Y_min); context.set(0, 0, interpolator); - final Rectangle era5Rect = new Rectangle(era5_X_min, era5_Y_min, 2, 2); - context.setEra5Region(era5Rect); - return context; } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPlugin.java index 8cb21acca..1d38b3e8d 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPlugin.java @@ -1,15 +1,40 @@ package com.bc.fiduceo.post.plugin.era5; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; import com.bc.fiduceo.util.JDomUtils; -import org.esa.snap.core.util.StringUtils; -import org.jdom.Attribute; -import org.jdom.Element; +import org.jdom2.Attribute; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.JDOMException; +import org.jdom2.input.SAXBuilder; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +import static org.esa.snap.core.util.StringUtils.isNotNullAndNotEmpty; public class Era5PostProcessingPlugin implements PostProcessingPlugin { - static Configuration createConfiguration(Element rootElement) { + static final String ERA5_POST_PROCESSING_GENERAL_INFO_XML = "era5-post-processing-general-info.xml"; + + private static final String TAG_NAME_SATELLITE_FIELDS = "satellite-fields"; + + private static final String ATT_NAME_UNITS = "units"; + private static final String ATT_NAME_LONG_NAME = "long_name"; + private static final String ATT_NAME_STANDARD_NAME = "standard_name"; + private static final String ATT_NAME_FILL_VALUE = "_FillValue"; + private static final String ATT_NAME_LENGTH = "length"; + + private static final Set KNOWN_ATTRIB_NAMES = new TreeSet<>( + Arrays.asList(ATT_NAME_UNITS, ATT_NAME_LONG_NAME, ATT_NAME_STANDARD_NAME, ATT_NAME_FILL_VALUE)); + + + static Configuration createConfiguration(Element rootElement, PostProcessingContext context) { final Configuration configuration = new Configuration(); final String nwpAuxDirValue = JDomUtils.getMandatoryChildTextTrim(rootElement, "nwp-aux-dir"); @@ -18,27 +43,41 @@ static Configuration createConfiguration(Element rootElement) { final Element era5CollectionElement = rootElement.getChild("era5-collection"); if (era5CollectionElement != null) { final String value = era5CollectionElement.getValue(); - if (StringUtils.isNotNullAndNotEmpty(value)) { + if (isNotNullAndNotEmpty(value)) { configuration.setEra5Collection(value); } } + final Element variableTranslationElement = rootElement.getChild("variable-translation"); + if (variableTranslationElement != null) { + final String value = variableTranslationElement.getTextTrim(); + final HashSet known = new HashSet<>(Arrays.asList("off", "false", "no", "dont", "not", "do not")); + configuration.setTranslateVariableNameToFileAccessName(!known.contains(value.toLowerCase())); + } + + parseGeneralizedInformation(configuration, context.getConfigDirectory()); parseSatelliteFields(rootElement, configuration); parseMatchupFields(rootElement, configuration); return configuration; } - private static void parseSatelliteFields(Element rootElement, Configuration configuration) { - final Element satelliteFieldsElement = rootElement.getChild("satellite-fields"); + + static void parseSatelliteFields(Element rootElement, Configuration configuration) { + final Element satelliteFieldsElement = rootElement.getChild(TAG_NAME_SATELLITE_FIELDS); if (satelliteFieldsElement != null) { - final SatelliteFieldsConfiguration satelliteFieldsConfiguration = new SatelliteFieldsConfiguration(); + final SatelliteFieldsConfiguration satelliteFieldsConfiguration; + if (configuration.getSatelliteFields() != null) { + satelliteFieldsConfiguration = configuration.getSatelliteFields(); + } else { + satelliteFieldsConfiguration = new SatelliteFieldsConfiguration(); + } final Element xDimElement = satelliteFieldsElement.getChild("x_dim"); if (xDimElement != null) { final Attribute nameElement = JDomUtils.getMandatoryAttribute(xDimElement, "name"); satelliteFieldsConfiguration.set_x_dim_name(nameElement.getValue()); - final Attribute lengthElement = JDomUtils.getMandatoryAttribute(xDimElement, "length"); + final Attribute lengthElement = JDomUtils.getMandatoryAttribute(xDimElement, ATT_NAME_LENGTH); satelliteFieldsConfiguration.set_x_dim(Integer.parseInt(lengthElement.getValue())); } @@ -46,7 +85,7 @@ private static void parseSatelliteFields(Element rootElement, Configuration conf if (yDimElement != null) { final Attribute nameElement = JDomUtils.getMandatoryAttribute(yDimElement, "name"); satelliteFieldsConfiguration.set_y_dim_name(nameElement.getValue()); - final Attribute lengthElement = JDomUtils.getMandatoryAttribute(yDimElement, "length"); + final Attribute lengthElement = JDomUtils.getMandatoryAttribute(yDimElement, ATT_NAME_LENGTH); satelliteFieldsConfiguration.set_y_dim(Integer.parseInt(lengthElement.getValue())); } @@ -54,13 +93,8 @@ private static void parseSatelliteFields(Element rootElement, Configuration conf if (zDimElement != null) { final Attribute nameElement = JDomUtils.getMandatoryAttribute(zDimElement, "name"); satelliteFieldsConfiguration.set_z_dim_name(nameElement.getValue()); - final Attribute lengthElement = zDimElement.getAttribute("length"); - if (lengthElement != null) { - satelliteFieldsConfiguration.set_z_dim(Integer.parseInt(lengthElement.getValue())); - } else { - // then we take all levels tb 2020-11-16 - satelliteFieldsConfiguration.set_z_dim(137); - } + final Attribute lengthElement = JDomUtils.getMandatoryAttribute(zDimElement, ATT_NAME_LENGTH); + satelliteFieldsConfiguration.set_z_dim(Integer.parseInt(lengthElement.getValue())); } final Element sensorRefElement = satelliteFieldsElement.getChild("sensor-ref"); @@ -68,69 +102,12 @@ private static void parseSatelliteFields(Element rootElement, Configuration conf satelliteFieldsConfiguration.setSensorRef(sensorRefElement.getValue()); } - final Element humidityElement = satelliteFieldsElement.getChild("an_ml_q"); - if (humidityElement != null) { - satelliteFieldsConfiguration.set_an_q_name(getElementValueTrimmed(humidityElement)); - } - - final Element temperatureElement = satelliteFieldsElement.getChild("an_ml_t"); - if (temperatureElement != null) { - satelliteFieldsConfiguration.set_an_t_name(getElementValueTrimmed(temperatureElement)); - } - - final Element ozoneElement = satelliteFieldsElement.getChild("an_ml_o3"); - if (ozoneElement != null) { - satelliteFieldsConfiguration.set_an_o3_name(getElementValueTrimmed(ozoneElement)); - } - - final Element pressureElement = satelliteFieldsElement.getChild("an_ml_lnsp"); - if (pressureElement != null) { - satelliteFieldsConfiguration.set_an_lnsp_name(getElementValueTrimmed(pressureElement)); - } - - final Element temp2metersElement = satelliteFieldsElement.getChild("an_sfc_t2m"); - if (temp2metersElement != null) { - satelliteFieldsConfiguration.set_an_t2m_name(getElementValueTrimmed(temp2metersElement)); - } - - final Element windUElement = satelliteFieldsElement.getChild("an_sfc_u10"); - if (windUElement != null) { - satelliteFieldsConfiguration.set_an_u10_name(getElementValueTrimmed(windUElement)); - } - - final Element windVElement = satelliteFieldsElement.getChild("an_sfc_v10"); - if (windVElement != null) { - satelliteFieldsConfiguration.set_an_v10_name(getElementValueTrimmed(windVElement)); - } - - final Element seaIceElement = satelliteFieldsElement.getChild("an_sfc_siconc"); - if (seaIceElement != null) { - satelliteFieldsConfiguration.set_an_siconc_name(getElementValueTrimmed(seaIceElement)); - } - - final Element surfPressElement = satelliteFieldsElement.getChild("an_sfc_msl"); - if (surfPressElement != null) { - satelliteFieldsConfiguration.set_an_msl_name(getElementValueTrimmed(surfPressElement)); - } - - final Element skinTempElement = satelliteFieldsElement.getChild("an_sfc_skt"); - if (skinTempElement != null) { - satelliteFieldsConfiguration.set_an_skt_name(getElementValueTrimmed(skinTempElement)); - } - - final Element sstElement = satelliteFieldsElement.getChild("an_sfc_sst"); - if (sstElement != null) { - satelliteFieldsConfiguration.set_an_sst_name(getElementValueTrimmed(sstElement)); - } - - final Element cloudElement = satelliteFieldsElement.getChild("an_sfc_tcc"); - if (cloudElement != null) { - satelliteFieldsConfiguration.set_an_tcc_name(getElementValueTrimmed(cloudElement)); - } - - final Element waterVaporElement = satelliteFieldsElement.getChild("an_sfc_tcwv"); - if (waterVaporElement != null) { - satelliteFieldsConfiguration.set_an_tcwv_name(getElementValueTrimmed(waterVaporElement)); + final Set varNameKeys = satelliteFieldsConfiguration.getVarNameKeys(); + for (String varNameKey : varNameKeys) { + final Element varNameElement = satelliteFieldsElement.getChild(varNameKey); + if (varNameElement != null) { + satelliteFieldsConfiguration.setVarName(varNameKey, getElementValueTrimmed(varNameElement)); + } } final Element era5TimeElement = satelliteFieldsElement.getChild("era5_time_variable"); @@ -258,13 +235,87 @@ private static void parseMatchupFields(Element rootElement, Configuration config } } + // package access for testing only se 2024-02-21 + static void parseGeneralizedInformation(Configuration config, Path configPath) { + final Path infoPath = configPath.resolve(ERA5_POST_PROCESSING_GENERAL_INFO_XML); + if (!Files.exists(infoPath)) { + return; + } + final SAXBuilder saxBuilder = new SAXBuilder(); + try (InputStream inputStream = Files.newInputStream(infoPath)) { + final Document document = saxBuilder.build(inputStream); + parseGeneralizedInformation(document.getRootElement(), config); + } catch (IOException e) { + throw new RuntimeException("Unable to create input stream from " + infoPath + ".", e); + } catch (JDOMException e) { + throw new RuntimeException("XML document " + infoPath + " could not be read in.", e); + } catch (Throwable e) { + throw new RuntimeException(e.getMessage() + " in " + infoPath, e); + } + } + + // package access for testing only se 2024-02-21 + static void parseGeneralizedInformation(Element root, Configuration config) throws Throwable { + if (root == null || !"era5".equals(root.getName())) { + throw new Throwable("Root tag expected"); + } + final Element satFieldsElem = root.getChild(TAG_NAME_SATELLITE_FIELDS); + if (satFieldsElem != null) { + final Map templateVariables = new LinkedHashMap<>(); + final List collections = satFieldsElem.getChildren("collection"); + for (Element collection : collections) { + final String collectionName = JDomUtils.getValueFromNameAttributeMandatory(collection).trim(); + final Element is3d_E = collection.getChild("is3d"); + final boolean is3d = is3d_E != null && "true".equalsIgnoreCase(is3d_E.getTextTrim()); + final List variables = collection.getChildren("var"); + for (Element var_E : variables) { + final HashMap attributes = new HashMap<>(); + final String varName = JDomUtils.getValueFromNameAttributeMandatory(var_E); + final List attribElems = var_E.getChildren("att"); + for (Element attrib_E : attribElems) { + final String attName = JDomUtils.getValueFromNameAttributeMandatory(attrib_E).trim(); + if (!KNOWN_ATTRIB_NAMES.contains(attName)) { + throw new Throwable("Unknown attribute name " + attName); + } + final String value = attrib_E.getTextTrim(); + if (isNotNullAndNotEmpty(attName) && isNotNullAndNotEmpty(value)) { + attributes.put(attName, value); + } + } + final String templateKey = collectionName + "_" + varName; + final TemplateVariable templateVariable = new TemplateVariable( + varName, + attributes.containsKey(ATT_NAME_UNITS) ? attributes.get(ATT_NAME_UNITS) : "~", + attributes.get(ATT_NAME_LONG_NAME), + attributes.get(ATT_NAME_STANDARD_NAME), + is3d); + if (attributes.containsKey(ATT_NAME_FILL_VALUE)) { + templateVariable.setFill_value(Float.valueOf(attributes.get(ATT_NAME_FILL_VALUE))); + } + templateVariables.put( + templateKey, + templateVariable + ); + } + } + if (!templateVariables.isEmpty()) { + SatelliteFieldsConfiguration satFields = config.getSatelliteFields(); + if (satFields == null) { + satFields = new SatelliteFieldsConfiguration(); + } + satFields.setGeneralizedVariables(templateVariables); + config.setSatelliteFields(satFields); + } + } + } + private static String getElementValueTrimmed(Element element) { return element.getValue().trim(); } @Override - public PostProcessing createPostProcessing(Element element) { - final Configuration configuration = createConfiguration(element); + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { + final Configuration configuration = createConfiguration(element, context); return new Era5PostProcessing(configuration); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/FieldsProcessor.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/FieldsProcessor.java index 656f4ff62..6ed6d0f89 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/FieldsProcessor.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/FieldsProcessor.java @@ -8,52 +8,14 @@ import java.awt.*; import java.io.IOException; -class FieldsProcessor { +import static com.bc.fiduceo.post.plugin.era5.Era5PostProcessing.DATA_ARRAY_WIDTH; - private static final int RASTER_WIDTH = 1440; +class FieldsProcessor { TemplateVariable createTemplate(String name, String units, String longName, String standardName, boolean is3d) { return new TemplateVariable(name, units, longName, standardName, is3d); } - static Array readSubset(int numLayers, Rectangle era5RasterPosition, VariableCache.CacheEntry cacheEntry) throws IOException, InvalidRangeException { - Array subset; - - final int maxRequestedX = era5RasterPosition.x + era5RasterPosition.width - 1; - if (era5RasterPosition.x < 0 || maxRequestedX >= RASTER_WIDTH) { - subset = readVariableDataOverlapped(numLayers, era5RasterPosition, cacheEntry.array); - } else { - subset = readVariableData(numLayers, era5RasterPosition, cacheEntry.array); - } - - return NetCDFUtils.scaleIfNecessary(cacheEntry.variable, subset); - } - - private static Array readVariableDataOverlapped(int numLayers, Rectangle era5RasterPosition, Array array) throws IOException, InvalidRangeException { - Array subset; - int xMin = 0; - int xMax; - int leftWidth; - int rightWidth; - if (era5RasterPosition.x < 0) { - xMax = RASTER_WIDTH + era5RasterPosition.x; // notabene: rasterposition is negative tb 2021-01-13 - leftWidth = era5RasterPosition.width + era5RasterPosition.x; - rightWidth = -era5RasterPosition.x; - } else { - xMax = era5RasterPosition.x; - rightWidth = RASTER_WIDTH - era5RasterPosition.x; - leftWidth = era5RasterPosition.width - rightWidth; - } - final Rectangle leftEraPos = new Rectangle(xMin, era5RasterPosition.y, leftWidth, era5RasterPosition.height); - final Array leftSubset = readVariableData(numLayers, leftEraPos, array); - - final Rectangle rightEraPos = new Rectangle(xMax, era5RasterPosition.y, rightWidth, era5RasterPosition.height); - final Array rightSubset = readVariableData(numLayers, rightEraPos, array); - - subset = mergeData(leftSubset, rightSubset, numLayers, era5RasterPosition, array); - return subset; - } - static Array mergeData(Array leftSubset, Array rightSubset, int numLayers, Rectangle era5RasterPosition, Array array) { final int rank = array.getRank(); final Array mergedArray; @@ -63,7 +25,7 @@ static Array mergeData(Array leftSubset, Array rightSubset, int numLayers, Recta final int xMax = era5RasterPosition.width + era5RasterPosition.x; mergeArrays_3D(leftSubset, rightSubset, era5RasterPosition, mergedArray, xMax); } else { - final int xMax = RASTER_WIDTH - era5RasterPosition.x; + final int xMax = DATA_ARRAY_WIDTH - era5RasterPosition.x; mergeArrays_3D(rightSubset, leftSubset, era5RasterPosition, mergedArray, xMax); } } else { @@ -72,7 +34,7 @@ static Array mergeData(Array leftSubset, Array rightSubset, int numLayers, Recta final int xMax = era5RasterPosition.width + era5RasterPosition.x; mergeArrays(leftSubset, rightSubset, era5RasterPosition, mergedArray, xMax); } else { - final int xMax = RASTER_WIDTH - era5RasterPosition.x; + final int xMax = DATA_ARRAY_WIDTH - era5RasterPosition.x; mergeArrays(rightSubset, leftSubset, era5RasterPosition, mergedArray, xMax); } } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/InterpolationContext.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/InterpolationContext.java index 8de815fb5..1a7f85377 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/InterpolationContext.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/InterpolationContext.java @@ -1,18 +1,33 @@ package com.bc.fiduceo.post.plugin.era5; -import java.awt.*; +import com.bc.fiduceo.core.IntRange; + +import java.util.ArrayList; +import java.util.List; + +import static com.bc.fiduceo.post.plugin.era5.Era5PostProcessing.DATA_ARRAY_WIDTH; class InterpolationContext { + private static final int EAST = 0; + private static final int WEST = 1; + private final BilinearInterpolator[][] interpolators; private final int width; private final int height; - private Rectangle era5Region; + private final IntRange yRange; + private final IntRange[] xRanges; + private boolean mustInitialise; - InterpolationContext(int x, int y) { - width = x; - height = y; - interpolators = new BilinearInterpolator[y][x]; + InterpolationContext(int width, int height) { + this.width = width; + this.height = height; + interpolators = new BilinearInterpolator[height][width]; + yRange = new IntRange(Integer.MAX_VALUE, Integer.MIN_VALUE); + xRanges = new IntRange[2]; + xRanges[0] = new IntRange(Integer.MAX_VALUE, Integer.MIN_VALUE); + xRanges[1] = null; + mustInitialise = true; } BilinearInterpolator get(int x, int y) { @@ -22,21 +37,119 @@ BilinearInterpolator get(int x, int y) { public void set(int x, int y, BilinearInterpolator interpolator) { checkBoundaries(x, y); - interpolators[y][x] = interpolator; } - Rectangle getEra5Region() { - return era5Region; + private void checkBoundaries(int x, int y) { + if (x < 0 || x >= width || y < 0 || y >= height) { + throw new IllegalArgumentException("Access interpolator out of raster: " + x + ", " + y); + } } - void setEra5Region(Rectangle era5Region) { - this.era5Region = era5Region; + public IntRange[] getXRanges() { + if (mustInitialise) { + initialize(); + } + return xRanges; } - private void checkBoundaries(int x, int y) { - if (x < 0 || x >= width || y < 0 || y >= height) { - throw new IllegalArgumentException("Access interpolator out of raster: " + x + ", " + y); + private void initialize() { + final List xStartPositions = new ArrayList<>(); + final List yStartPositions = new ArrayList<>(); + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + final BilinearInterpolator interpolator = interpolators[y][x]; + if (interpolator != null) { + xStartPositions.add(interpolator.getXMin()); + yStartPositions.add(interpolator.getYMin()); + } + } + } + + if (xStartPositions.isEmpty() || yStartPositions.isEmpty()) { + return; + } + + xStartPositions.sort(Integer::compare); + final int largestX = xStartPositions.get(xStartPositions.size() - 1); + if (largestX == DATA_ARRAY_WIDTH - 1) { + // we are overlapping antimeridian, split xPositions int two groups, one east, one west + xRanges[WEST] = new IntRange(); + xRanges[WEST].setMin(0); // western section starts at anti-meridian + xRanges[EAST].setMax(DATA_ARRAY_WIDTH - 1); // eastern section ends at anti-meridian + int activeRange = WEST; + boolean switchRange = false; + for (int i = 0; i < xStartPositions.size() - 1; i++) { + final int currentX = xStartPositions.get(i); + final int delta = xStartPositions.get(i + 1) - currentX; + if (delta > 1000) { + switchRange = true; + } + + if (activeRange == WEST) { + if (currentX > xRanges[WEST].getMax()) { + xRanges[WEST].setMax(currentX); + } + } + if (activeRange == EAST) { + if (currentX < xRanges[EAST].getMin()) { + xRanges[EAST].setMin(currentX); + } + } + + if (switchRange) { + activeRange = EAST; + switchRange = false; + } + } + // don't forget to increase. All x-es are minimum values, we need one x more to be able to interpolate + xRanges[WEST].setMax(xRanges[WEST].getMax() + 1); + } else { + xRanges[0].setMin(xStartPositions.get(0)); + xRanges[0].setMax(xStartPositions.get(xStartPositions.size() - 1) + 1); + xRanges[1] = null; + } + + yStartPositions.sort(Integer::compareTo); + yRange.setMin(yStartPositions.get(0)); + yRange.setMax(yStartPositions.get(yStartPositions.size() - 1) + 1); + + setRelativeInterpolatorOffsets(); + + mustInitialise = false; + } + + private void setRelativeInterpolatorOffsets() { + final int yMinValue = yRange.getMin(); + + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + final BilinearInterpolator interpolator = interpolators[y][x]; + if (interpolator == null) { + continue; + } + + interpolator.setRelYMin(interpolator.getYMin() - yMinValue); + + int xMin = interpolator.getXMin(); + if (xRanges[0].contains(xMin)) { + int xMinValue = xRanges[0].getMin(); + interpolator.setRelXMin(interpolator.getXMin() - xMinValue); + } else if (xRanges.length > 1) { + final int xOffset = xRanges[0].getLength(); + if (xRanges[1].contains(xMin)) { + int xMinValue = xRanges[1].getMin(); + interpolator.setRelXMin(interpolator.getXMin() - xMinValue + xOffset); + } + } + } + } + } + + public IntRange getYRange() { + if (mustInitialise) { + initialize(); } + return yRange; } } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java index c0b5a82a7..11a4bf883 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java @@ -1,6 +1,6 @@ package com.bc.fiduceo.post.plugin.era5; -import com.bc.fiduceo.FiduceoConstants; +import com.bc.fiduceo.reader.ReaderUtils; import com.bc.fiduceo.util.NetCDFUtils; import ucar.ma2.Array; import ucar.ma2.DataType; @@ -9,11 +9,11 @@ import ucar.nc2.Dimension; import ucar.nc2.*; -import java.awt.*; import java.io.IOException; import java.util.List; import java.util.*; +import static com.bc.fiduceo.post.plugin.era5.Era5PostProcessing.DATA_ARRAY_WIDTH; import static com.bc.fiduceo.post.plugin.era5.VariableUtils.*; class MatchupFields extends FieldsProcessor { @@ -68,7 +68,7 @@ void prepare(MatchupFieldsConfiguration matchupFieldsConfig, NetcdfFile reader, } void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException { - final Era5Archive era5Archive = new Era5Archive(config.getNWPAuxDir(), collection); + final Era5Archive era5Archive = new Era5Archive(config, collection); final MatchupFieldsConfiguration matchupConfig = config.getMatchupFields(); // allocate cache large enough to hold the time-series for one Era-5 variable @@ -104,6 +104,7 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t // iterate over matchups for (final String variableKey : variableKeys) { + final float fillValue = variables.get(variableKey).getFillValue(); final Array targetArray = targetArrays.get(variableKey); final Index targetIndex = targetArray.getIndex(); @@ -114,7 +115,6 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t final Array latLayer = latArray.section(nwpOffset, nwpShape).copy(); final InterpolationContext interpolationContext = Era5PostProcessing.getInterpolationContext(lonLayer, latLayer); - final Rectangle layerRegion = interpolationContext.getEra5Region(); // iterate over time stamps for (int t = 0; t < numTimeSteps; t++) { @@ -123,32 +123,35 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t final int timeStamp = targetTimeArray.getInt(timeIndex); if (VariableUtils.isTimeFill(timeStamp)) { - targetArray.setFloat(targetIndex, TemplateVariable.getFillValue()); + targetArray.setFloat(targetIndex, fillValue); continue; } - VariableCache.CacheEntry cacheEntry = variableCache.get(variableKey, timeStamp); + final Variable variable = variableCache.get(variableKey, timeStamp); + final double scaleFactor = NetCDFUtils.getScaleFactor(variable); + final double offset = NetCDFUtils.getOffset(variable); + final boolean mustScale = ReaderUtils.mustScale(scaleFactor, offset); - // read and get rid of fake z-dimension - final Array subset = readSubset(1, layerRegion, cacheEntry); - final Index subsetIndex = subset.getIndex(); final BilinearInterpolator bilinearInterpolator = interpolationContext.get(0, 0); if (bilinearInterpolator == null) { - targetArray.setFloat(targetIndex, TemplateVariable.getFillValue()); + targetArray.setFloat(targetIndex, fillValue); continue; } - subsetIndex.set(0, 0); - final float c00 = subset.getFloat(subsetIndex); - - subsetIndex.set(0, 1); - final float c10 = subset.getFloat(subsetIndex); - - subsetIndex.set(1, 0); - final float c01 = subset.getFloat(subsetIndex); - - subsetIndex.set(1, 1); - final float c11 = subset.getFloat(subsetIndex); + final int offsetX = bilinearInterpolator.getXMin(); + final int offsetX1 = (offsetX + 1) % DATA_ARRAY_WIDTH; + final int offsetY = bilinearInterpolator.getYMin(); + + Array leftSide = variable.read(new int[]{0, offsetY, offsetX}, new int[]{1, 2, 1}).reduce(); + Array rightSide = variable.read(new int[]{0, offsetY, offsetX1}, new int[]{1, 2, 1}).reduce(); + if (mustScale) { + leftSide = NetCDFUtils.scale(leftSide, scaleFactor, offset); + rightSide = NetCDFUtils.scale(rightSide, scaleFactor, offset); + } + final float c00 = leftSide.getFloat(0); + final float c10 = rightSide.getFloat(0); + final float c01 = leftSide.getFloat(1); + final float c11 = rightSide.getFloat(1); final double interpolated = bilinearInterpolator.interpolate(c00, c10, c01, c11); targetArray.setFloat(targetIndex, (float) interpolated); diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java index 9e992a588..e29d126c5 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java @@ -1,20 +1,22 @@ package com.bc.fiduceo.post.plugin.era5; import com.bc.fiduceo.FiduceoConstants; +import com.bc.fiduceo.core.IntRange; +import com.bc.fiduceo.log.FiduceoLogger; +import com.bc.fiduceo.reader.ReaderUtils; import com.bc.fiduceo.util.NetCDFUtils; import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.Index; import ucar.ma2.InvalidRangeException; import ucar.nc2.*; -import ucar.nc2.Dimension; -import java.awt.*; import java.io.IOException; import java.util.*; -import java.util.List; +import java.util.logging.Logger; import static com.bc.fiduceo.post.plugin.era5.VariableUtils.*; +import static com.bc.fiduceo.post.util.PPUtils.convertToFitTheRangeMinus180to180; class SatelliteFields extends FieldsProcessor { @@ -44,7 +46,7 @@ void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, Ne void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException { final SatelliteFieldsConfiguration satFieldsConfig = config.getSatelliteFields(); final int numLayers = satFieldsConfig.get_z_dim(); - final Era5Archive era5Archive = new Era5Archive(config.getNWPAuxDir(), collection); + final Era5Archive era5Archive = new Era5Archive(config, collection); final VariableCache variableCache = new VariableCache(era5Archive, 52); // 4 * 13 variables tb 2020-11-25 try { @@ -62,6 +64,7 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t // + scale if necessary final com.bc.fiduceo.core.Dimension geoDimension = new com.bc.fiduceo.core.Dimension("geoloc", satFieldsConfig.get_x_dim(), satFieldsConfig.get_y_dim()); final Array lonArray = readGeolocationVariable(geoDimension, reader, satFieldsConfig.get_longitude_variable_name()); + convertToFitTheRangeMinus180to180(lonArray); final Array latArray = readGeolocationVariable(geoDimension, reader, satFieldsConfig.get_latitude_variable_name()); // prepare data @@ -85,12 +88,7 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t final Array lonLayer = lonArray.sectionNoReduce(nwpOffset, nwpShape, nwpStride).reduce(0).copy(); final Array latLayer = latArray.sectionNoReduce(nwpOffset, nwpShape, nwpStride).reduce(0).copy(); - final int[] shape = lonLayer.getShape(); - final int width = shape[1]; - final int height = shape[0]; - final InterpolationContext interpolationContext = Era5PostProcessing.getInterpolationContext(lonLayer, latLayer); - final Rectangle layerRegion = interpolationContext.getEra5Region(); timeIndex.set(m); final int era5Time = era5TimeArray.getInt(timeIndex); @@ -103,82 +101,85 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t // - store to target raster final Set variableKeys = variables.keySet(); for (final String variableKey : variableKeys) { - VariableCache.CacheEntry cacheEntry = variableCache.get(variableKey, era5Time); - final Array subset = readSubset(numLayers, layerRegion, cacheEntry); - final Index subsetIndex = subset.getIndex(); + final float fillValue = variables.get(variableKey).getFillValue(); + final Variable variable = variableCache.get(variableKey, era5Time); final Array targetArray = targetArrays.get(variableKey); final Index targetIndex = targetArray.getIndex(); - final int rank = subset.getRank(); - if (rank == 2) { + final int rank = variable.getRank(); + final Array era5Data = loadEra5Data(variable, interpolationContext, numLayers); + final Index era5Index = era5Data.getIndex(); + + + final int[] shape = lonLayer.getShape(); + final int width = shape[1]; + final int height = shape[0]; + + if (rank == 3) { for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { targetIndex.set(m, y, x); if (isTimeFill) { - targetArray.setFloat(targetIndex, TemplateVariable.getFillValue()); + targetArray.setFloat(targetIndex, fillValue); continue; } final BilinearInterpolator interpolator = interpolationContext.get(x, y); if (interpolator == null) { - targetArray.setFloat(targetIndex, TemplateVariable.getFillValue()); + targetArray.setFloat(targetIndex, fillValue); continue; } - final int offsetX = interpolator.getXMin() - layerRegion.x; - final int offsetY = interpolator.getYMin() - layerRegion.y; - - subsetIndex.set(offsetY, offsetX); - final float c00 = subset.getFloat(subsetIndex); - - subsetIndex.set(offsetY, offsetX + 1); - final float c10 = subset.getFloat(subsetIndex); - - subsetIndex.set(offsetY + 1, offsetX); - final float c01 = subset.getFloat(subsetIndex); - - subsetIndex.set(offsetY + 1, offsetX + 1); - final float c11 = subset.getFloat(subsetIndex); + // calculate offsets into era5 data + final int era5X = interpolator.getRelXMin(); + final int era5Y = interpolator.getRelYMin(); + // read c00 -> c11 directly from era5 data array + era5Index.set(0, era5Y, era5X); + final float c00 = era5Data.getFloat(era5Index); + era5Index.set(0, era5Y, era5X + 1); + final float c10 = era5Data.getFloat(era5Index); + era5Index.set(0, era5Y + 1, era5X); + final float c01 = era5Data.getFloat(era5Index); + era5Index.set(0, era5Y + 1, era5X + 1); + final float c11 = era5Data.getFloat(era5Index); final double interpolate = interpolator.interpolate(c00, c10, c01, c11); targetArray.setFloat(targetIndex, (float) interpolate); } } - } else if (rank == 3) { + } else if (rank == 4) { for (int z = 0; z < numLayers; z++) { for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { targetIndex.set(m, z, y, x); if (isTimeFill) { - targetArray.setFloat(targetIndex, TemplateVariable.getFillValue()); + targetArray.setFloat(targetIndex, fillValue); continue; } final BilinearInterpolator interpolator = interpolationContext.get(x, y); if (interpolator == null) { - targetArray.setFloat(targetIndex, TemplateVariable.getFillValue()); + targetArray.setFloat(targetIndex, fillValue); continue; } - final int offsetX = interpolator.getXMin() - layerRegion.x; - final int offsetY = interpolator.getYMin() - layerRegion.y; - - subsetIndex.set(z, offsetY, offsetX); - final float c00 = subset.getFloat(subsetIndex); - - subsetIndex.set(z, offsetY, offsetX + 1); - final float c10 = subset.getFloat(subsetIndex); - - subsetIndex.set(z, offsetY + 1, offsetX); - final float c01 = subset.getFloat(subsetIndex); - - subsetIndex.set(z, offsetY + 1, offsetX + 1); - final float c11 = subset.getFloat(subsetIndex); - - final double interpolate = interpolator.interpolate(c00, c01, c10, c11); + // calculate offsets into era5 data + final int era5X = interpolator.getRelXMin(); + final int era5Y = interpolator.getRelYMin(); + // read c00 -> c11 directly from era5 data array + era5Index.set(0, z, era5Y, era5X); + final float c00 = era5Data.getFloat(era5Index); + era5Index.set(0, z, era5Y, era5X + 1); + final float c10 = era5Data.getFloat(era5Index); + era5Index.set(0, z, era5Y + 1, era5X); + final float c01 = era5Data.getFloat(era5Index); + era5Index.set(0, z, era5Y + 1, era5X + 1); + final float c11 = era5Data.getFloat(era5Index); + + final double interpolate = interpolator.interpolate(c00, c10, c01, c11); targetArray.setFloat(targetIndex, (float) interpolate); } @@ -203,6 +204,61 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t } } + static Array loadEra5Data(Variable variable, InterpolationContext interpolationContext, int numLayers) throws InvalidRangeException, IOException { + // load the required ERA5 variable data + final double scaleFactor = NetCDFUtils.getScaleFactor(variable); + final double offset = NetCDFUtils.getOffset(variable); + + // request xRanges + final IntRange[] xRanges = interpolationContext.getXRanges(); + final IntRange yRange = interpolationContext.getYRange(); + if (xRanges[1] == null) { + return readFullEra5Array(variable, numLayers, yRange, xRanges[0], scaleFactor, offset); + } else { + final Array left = readFullEra5Array(variable, numLayers, yRange, xRanges[0], scaleFactor, offset); + final Array right = readFullEra5Array(variable, numLayers, yRange, xRanges[1], scaleFactor, offset); + + return ArrayUtils.mergeAlongX(left, right); + } + } + + private static Array readFullEra5Array(Variable variable, int numLayers, IntRange yRange, IntRange xRange, double scaleFactor, double offset) throws IOException, InvalidRangeException { + final int rank = variable.getRank(); + final int[] offsets; + final int[] shape; + + final int yMin = yRange.getMin(); + final int xMin = xRange.getMin(); + int yLength = yRange.getLength(); + int xlength = xRange.getLength(); + + if (rank == 3) { + offsets = new int[]{0, yMin, xMin}; + shape = new int[]{1, yLength, xlength}; + } else if (rank == 4) { + offsets = new int[]{0, 0, yMin, xMin}; + shape = new int[]{1, numLayers, yLength, xlength}; + } else { + throw new IllegalStateException("unsupport input data rank"); + } + + Array era5Data; + try { + era5Data = variable.read(offsets, shape); + if (ReaderUtils.mustScale(scaleFactor, offset)) { + era5Data = NetCDFUtils.scale(era5Data, scaleFactor, offset); + } + } catch (Exception e) { + final Logger logger = FiduceoLogger.getLogger(); + logger.severe("Unable to read: " + variable.getFullName()); + logger.severe("offsets: " + yMin + ", " + xMin); + logger.severe("shape: " + yLength + ", " + xlength); + throw e; + } + + return era5Data; + } + private void addTimeVariable(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFileWriter writer) { final String nwp_time_variable_name = satFieldsConfig.get_nwp_time_variable_name(); final Variable variable = writer.addVariable(nwp_time_variable_name, DataType.INT, FiduceoConstants.MATCHUP_COUNT); @@ -243,22 +299,30 @@ void setDimensions(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFileWrite } Map getVariables(SatelliteFieldsConfiguration configuration) { - final HashMap variablesMap = new HashMap<>(); - - variablesMap.put("an_ml_q", createTemplate(configuration.get_an_q_name(), "kg kg**-1", "Specific humidity", "specific_humidity", true)); - variablesMap.put("an_ml_t", createTemplate(configuration.get_an_t_name(), "K", "Temperature", "air_temperature", true)); - variablesMap.put("an_ml_o3", createTemplate(configuration.get_an_o3_name(), "kg kg**-1", "Ozone mass mixing ratio", null, true)); - variablesMap.put("an_ml_lnsp", createTemplate(configuration.get_an_lnsp_name(), "~", "Logarithm of surface pressure", null, false)); - variablesMap.put("an_sfc_t2m", createTemplate(configuration.get_an_t2m_name(), "K", "2 metre temperature", null, false)); - variablesMap.put("an_sfc_u10", createTemplate(configuration.get_an_u10_name(), "m s**-1", "10 metre U wind component", null, false)); - variablesMap.put("an_sfc_v10", createTemplate(configuration.get_an_v10_name(), "m s**-1", "10 metre V wind component", null, false)); - variablesMap.put("an_sfc_siconc", createTemplate(configuration.get_an_siconc_name(), "(0 - 1)", "Sea ice area fraction", "sea_ice_area_fraction", false)); - variablesMap.put("an_sfc_msl", createTemplate(configuration.get_an_msl_name(), "Pa", "Mean sea level pressure", "air_pressure_at_mean_sea_level", false)); - variablesMap.put("an_sfc_skt", createTemplate(configuration.get_an_skt_name(), "K", "Skin temperature", null, false)); - variablesMap.put("an_sfc_sst", createTemplate(configuration.get_an_sst_name(), "K", "Sea surface temperature", null, false)); - variablesMap.put("an_sfc_tcc", createTemplate(configuration.get_an_tcc_name(), "(0 - 1)", "Total cloud cover", "cloud_area_fraction", false)); - variablesMap.put("an_sfc_tcwv", createTemplate(configuration.get_an_tcwv_name(), "kg m**-2", "Total column water vapour", "lwe_thickness_of_atmosphere_mass_content_of_water_vapor", false)); - - return variablesMap; + final Map generalizedVariables = configuration.getGeneralizedVariables(); + if (generalizedVariables != null) { + for (String key : generalizedVariables.keySet()) { + generalizedVariables.get(key).setName(configuration.getVarName(key)); + } + return generalizedVariables; + } else { + final HashMap variablesMap = new HashMap<>(); + + variablesMap.put("an_ml_q", createTemplate(configuration.getVarName("an_ml_q"), "kg kg**-1", "Specific humidity", "specific_humidity", true)); + variablesMap.put("an_ml_t", createTemplate(configuration.getVarName("an_ml_t"), "K", "Temperature", "air_temperature", true)); + variablesMap.put("an_ml_o3", createTemplate(configuration.getVarName("an_ml_o3"), "kg kg**-1", "Ozone mass mixing ratio", null, true)); + variablesMap.put("an_ml_lnsp", createTemplate(configuration.getVarName("an_ml_lnsp"), "~", "Logarithm of surface pressure", null, false)); + variablesMap.put("an_sfc_msl", createTemplate(configuration.getVarName("an_sfc_msl"), "Pa", "Mean sea level pressure", "air_pressure_at_mean_sea_level", false)); + variablesMap.put("an_sfc_tcc", createTemplate(configuration.getVarName("an_sfc_tcc"), "(0 - 1)", "Total cloud cover", "cloud_area_fraction", false)); + variablesMap.put("an_sfc_t2m", createTemplate(configuration.getVarName("an_sfc_t2m"), "K", "2 metre temperature", null, false)); + variablesMap.put("an_sfc_u10", createTemplate(configuration.getVarName("an_sfc_u10"), "m s**-1", "10 metre U wind component", null, false)); + variablesMap.put("an_sfc_v10", createTemplate(configuration.getVarName("an_sfc_v10"), "m s**-1", "10 metre V wind component", null, false)); + variablesMap.put("an_sfc_siconc", createTemplate(configuration.getVarName("an_sfc_siconc"), "(0 - 1)", "Sea ice area fraction", "sea_ice_area_fraction", false)); + variablesMap.put("an_sfc_skt", createTemplate(configuration.getVarName("an_sfc_skt"), "K", "Skin temperature", null, false)); + variablesMap.put("an_sfc_sst", createTemplate(configuration.getVarName("an_sfc_sst"), "K", "Sea surface temperature", null, false)); + variablesMap.put("an_sfc_tcwv", createTemplate(configuration.getVarName("an_sfc_tcwv"), "kg m**-2", "Total column water vapour", "lwe_thickness_of_atmosphere_mass_content_of_water_vapor", false)); + + return variablesMap; + } } } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsConfiguration.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsConfiguration.java index 73c1c31ee..721a1be5e 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsConfiguration.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsConfiguration.java @@ -2,23 +2,14 @@ import org.esa.snap.core.util.StringUtils; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + class SatelliteFieldsConfiguration extends FieldsConfiguration { private static final String SENSOR_REF = "{sensor-ref}"; - - private String an_q_name; - private String an_t_name; - private String an_o3_name; - private String an_lnsp_name; - private String an_siconc_name; - private String an_t2m_name; - private String an_u10_name; - private String an_v10_name; - private String an_msl_name; - private String an_skt_name; - private String an_sst_name; - private String an_tcc_name; - private String an_tcwv_name; + private final HashMap variableNames; private int x_dim; private int y_dim; @@ -33,21 +24,23 @@ class SatelliteFieldsConfiguration extends FieldsConfiguration { private String time_variable_name; private String sensorRef; + private Map generalizedVariables; SatelliteFieldsConfiguration() { - an_q_name = "nwp_q"; - an_t_name = "nwp_t"; - an_o3_name = "nwp_o3"; - an_lnsp_name = "nwp_lnsp"; - an_siconc_name = "nwp_siconc"; - an_t2m_name = "nwp_t2m"; - an_u10_name = "nwp_u10"; - an_v10_name = "nwp_v10"; - an_msl_name = "nwp_msl"; - an_skt_name = "nwp_skt"; - an_sst_name = "nwp_sst"; - an_tcc_name = "nwp_tcc"; - an_tcwv_name = "nwp_tcwv"; + variableNames = new HashMap<>(); + variableNames.put("an_ml_q", "nwp_q"); + variableNames.put("an_ml_t", "nwp_t"); + variableNames.put("an_ml_o3", "nwp_o3"); + variableNames.put("an_ml_lnsp", "nwp_lnsp"); + variableNames.put("an_sfc_siconc", "nwp_siconc"); + variableNames.put("an_sfc_t2m", "nwp_t2m"); + variableNames.put("an_sfc_u10", "nwp_u10"); + variableNames.put("an_sfc_v10", "nwp_v10"); + variableNames.put("an_sfc_msl", "nwp_msl"); + variableNames.put("an_sfc_skt", "nwp_skt"); + variableNames.put("an_sfc_sst", "nwp_sst"); + variableNames.put("an_sfc_tcc", "nwp_tcc"); + variableNames.put("an_sfc_tcwv", "nwp_tcwv"); sensorRef = null; @@ -56,108 +49,16 @@ class SatelliteFieldsConfiguration extends FieldsConfiguration { z_dim = -1; } - String get_an_q_name() { - return expand(an_q_name); - } - - void set_an_q_name(String an_q_name) { - this.an_q_name = an_q_name; - } - - String get_an_t_name() { - return expand(an_t_name); - } - - void set_an_t_name(String an_t_name) { - this.an_t_name = an_t_name; + Set getVarNameKeys() { + return variableNames.keySet(); } - String get_an_o3_name() { - return expand(an_o3_name); + String getVarName(String key) { + return expand(variableNames.get(key)); } - void set_an_o3_name(String an_o3_name) { - this.an_o3_name = an_o3_name; - } - - String get_an_lnsp_name() { - return expand(an_lnsp_name); - } - - void set_an_lnsp_name(String an_lnsp_name) { - this.an_lnsp_name = an_lnsp_name; - } - - String get_an_t2m_name() { - return expand(an_t2m_name); - } - - void set_an_t2m_name(String an_t2m_name) { - this.an_t2m_name = an_t2m_name; - } - - String get_an_siconc_name() { - return expand(an_siconc_name); - } - - void set_an_siconc_name(String an_siconc_name) { - this.an_siconc_name = an_siconc_name; - } - - String get_an_u10_name() { - return expand(an_u10_name); - } - - void set_an_u10_name(String an_u10_name) { - this.an_u10_name = an_u10_name; - } - - String get_an_v10_name() { - return expand(an_v10_name); - } - - void set_an_v10_name(String an_v10_name) { - this.an_v10_name = an_v10_name; - } - - String get_an_msl_name() { - return expand(an_msl_name); - } - - void set_an_msl_name(String an_msl_name) { - this.an_msl_name = an_msl_name; - } - - String get_an_skt_name() { - return expand(an_skt_name); - } - - void set_an_skt_name(String an_skt_name) { - this.an_skt_name = an_skt_name; - } - - String get_an_sst_name() { - return expand(an_sst_name); - } - - void set_an_sst_name(String an_sst_name) { - this.an_sst_name = an_sst_name; - } - - String get_an_tcc_name() { - return expand(an_tcc_name); - } - - void set_an_tcc_name(String an_tcc_name) { - this.an_tcc_name = an_tcc_name; - } - - String get_an_tcwv_name() { - return expand(an_tcwv_name); - } - - void set_an_tcwv_name(String an_tcwv_name) { - this.an_tcwv_name = an_tcwv_name; + void setVarName(String key, String name) { + variableNames.put(key, name); } int get_x_dim() { @@ -254,6 +155,15 @@ void verify() { throw new IllegalArgumentException("dimensions incorrect: x:" + x_dim + " y:" + y_dim); } + if (z_dim_name != null) { + if (z_dim_name.isEmpty()) { + throw new IllegalArgumentException("z dimension name not configured"); + } + if (z_dim < 1) { + throw new IllegalArgumentException("dimension incorrect: z:" + z_dim); + } + } + if (StringUtils.isNullOrEmpty(x_dim_name)) { throw new IllegalArgumentException("x dimension name not configured"); } @@ -262,10 +172,6 @@ void verify() { throw new IllegalArgumentException("y dimension name not configured"); } - if (StringUtils.isNullOrEmpty(z_dim_name)) { - throw new IllegalArgumentException("z dimension name not configured"); - } - if (StringUtils.isNullOrEmpty(nwp_time_variable_name)) { throw new IllegalArgumentException("era-5 time variable name not configured"); } @@ -284,6 +190,17 @@ void verify() { } private String expand(String variableName) { - return expand(variableName, SENSOR_REF, sensorRef); + return expand(variableName, SENSOR_REF, getSensorRef()); + } + + public void setGeneralizedVariables(Map generalizedVariables) { + this.generalizedVariables = generalizedVariables; + for (String key : generalizedVariables.keySet()) { + setVarName(key, "nwp_" + generalizedVariables.get(key).getName()); + } + } + + public Map getGeneralizedVariables() { + return generalizedVariables; } } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/TemplateVariable.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/TemplateVariable.java index 42113fae7..d4aeb920b 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/TemplateVariable.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/TemplateVariable.java @@ -6,22 +6,33 @@ class TemplateVariable { private static final float FILL_VALUE = NetCDFUtils.getDefaultFillValue(float.class).floatValue(); - private final String name; + private String name; private final String units; private final String longName; private final String standardName; private final boolean is3d; + private float fill_value; + TemplateVariable(String name, String units, String longName, String standardName, boolean is3d) { this.name = name; this.units = units; this.longName = longName; this.standardName = standardName; this.is3d = is3d; + this.fill_value = FILL_VALUE; + } + + public void setFill_value(float fill_value) { + this.fill_value = fill_value; } - static float getFillValue() { - return FILL_VALUE; + float getFillValue() { + return fill_value; + } + + public void setName(String name) { + this.name = name; } String getName() { diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableCache.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableCache.java index 773a9051b..afa129e72 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableCache.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableCache.java @@ -1,117 +1,127 @@ package com.bc.fiduceo.post.plugin.era5; -import org.esa.snap.core.util.io.FileUtils; -import ucar.ma2.Array; +import org.apache.commons.lang3.StringUtils; import ucar.nc2.NetcdfFile; import ucar.nc2.Variable; -import java.io.File; import java.io.IOException; -import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; -import java.util.Set; class VariableCache { private final Era5Archive archive; - private final HashMap cache; - private final int cacheSize; + // We use a LRUCacheWithListener here so that the oldest entry will be removed automatically. + // This requires a listener to be added to this LRUCache. + private final LRUCacheWithListener> cacheTimeStamp; VariableCache(Era5Archive archive, int cacheSize) { this.archive = archive; - this.cacheSize = cacheSize; - cache = new HashMap<>(); + cacheTimeStamp = new LRUCacheWithListener<>(cacheSize); + cacheTimeStamp.setListener((key, value) -> freeContainers(value)); } - CacheEntry get(String variableKey, int era5TimeStamp) throws IOException { + Variable get(String variableKey, int era5TimeStamp) throws IOException { final CacheContainer cacheContainer = getCacheContainer(variableKey, era5TimeStamp); - return cacheContainer.cacheEntry; + if (cacheContainer == null) { + return createCacheContainer(variableKey, era5TimeStamp).variable; + } + return cacheContainer.variable; } private CacheContainer getCacheContainer(String variableKey, int era5TimeStamp) throws IOException { + final Map cacheContainer = cacheTimeStamp.get(era5TimeStamp); + if (cacheContainer != null) { + return cacheContainer.get(variableKey); + } + return null; + } + + private CacheContainer createCacheContainer(String variableKey, int era5TimeStamp) throws IOException { final String filePath = archive.get(variableKey, era5TimeStamp); final String variableName = getVariableName(variableKey); - final String key = FileUtils.getFilenameWithoutExtension(new File(filePath)); - - CacheContainer cacheContainer = cache.get(key); - if (cacheContainer == null) { - final NetcdfFile netcdfFile = NetcdfFile.open(filePath); - final Variable variable = netcdfFile.findVariable(variableName); - if (variable == null) { - throw new IOException("variable not found: " + variableName + " " + filePath); - } - if (cache.size() == cacheSize) { - removeOldest(); - } - final Array array = variable.read().reduce(); - cacheContainer = new CacheContainer(variable, netcdfFile, array, System.currentTimeMillis()); - cache.put(key, cacheContainer); + final NetcdfFile netcdfFile = NetcdfFile.open(filePath); + final Variable variable = netcdfFile.findVariable(variableName); + if (variable == null) { + throw new IOException("variable not found: " + variableName + " " + filePath); } - cacheContainer.lastAccess = System.currentTimeMillis(); + CacheContainer cacheContainer = new CacheContainer(variable, netcdfFile); + final Map cacheContainerMap; + if (cacheTimeStamp.containsKey(era5TimeStamp)) { + cacheContainerMap = cacheTimeStamp.get(era5TimeStamp); + } else { + cacheContainerMap = new HashMap<>(); + cacheTimeStamp.put(era5TimeStamp, cacheContainerMap); + } + cacheContainerMap.put(variableKey, cacheContainer); return cacheContainer; } - void close() throws IOException { - final Collection cacheEntries = cache.values(); - for (CacheContainer cacheContainer : cacheEntries) { - cacheContainer.netcdfFile.close(); - cacheContainer.netcdfFile = null; + void close() { + for (Map cacheContainerMap : cacheTimeStamp.values()) { + freeContainers(cacheContainerMap); } - - cache.clear(); + cacheTimeStamp.clear(); } private String getVariableName(String variableKey) { - final int cutPoint = variableKey.lastIndexOf("_"); - return variableKey.substring(cutPoint + 1, variableKey.length()); + final int cutPoint = StringUtils.ordinalIndexOf(variableKey, "_", 2); + return variableKey.substring(cutPoint + 1); } - private void removeOldest() throws IOException { - long minTime = Long.MAX_VALUE; - String toRemove = null; - CacheContainer entryToRemove = null; - final Set> cacheEntries = cache.entrySet(); - for (Map.Entry cacheMapEntry : cacheEntries) { - final CacheContainer cacheContainer = cacheMapEntry.getValue(); - if (cacheContainer.lastAccess < minTime) { - minTime = cacheContainer.lastAccess; - toRemove = cacheMapEntry.getKey(); - entryToRemove = cacheContainer; + private void freeContainers(Map cacheContainers) { + for (CacheContainer container : cacheContainers.values()) { + container.variable = null; + try { + container.netcdfFile.close(); + container.netcdfFile = null; + } catch (IOException e) { + throw new RuntimeException(e); } } + cacheContainers.clear(); + } - if (entryToRemove != null) { - entryToRemove.cacheEntry = null; - entryToRemove.netcdfFile.close(); + private static class LRUCacheWithListener extends LinkedHashMap { + private int capacity; + private RemovalListener listener; + private boolean listenerWasSet = false; - cache.remove(toRemove); + public LRUCacheWithListener(int capacity) { + super(capacity + 1, 1.0f, true); // Pass 'true' for accessOrder. + this.capacity = capacity; } - } - static class CacheEntry { + public interface RemovalListener { + void onRemove(K key, V value); + } - final Variable variable; - final Array array; + public void setListener(RemovalListener listener) { + this.listener = listener; + listenerWasSet = listener != null; + } - CacheEntry(Variable variable, Array array) { - this.variable = variable; - this.array = array; + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + boolean shouldRemove = this.size() > capacity; + if (listenerWasSet && shouldRemove) { + listener.onRemove(eldest.getKey(), eldest.getValue()); + } + return shouldRemove; } } private static class CacheContainer { - CacheEntry cacheEntry; + Variable variable; NetcdfFile netcdfFile; - long lastAccess; - CacheContainer(Variable variable, NetcdfFile netcdfFile, Array array, long lastAccess) { - this.cacheEntry = new CacheEntry(variable, array); + CacheContainer(Variable variable, NetcdfFile netcdfFile) { + this.variable = variable; this.netcdfFile = netcdfFile; - this.lastAccess = lastAccess; } } } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableUtils.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableUtils.java index cc53629cc..8322f3ed6 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableUtils.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableUtils.java @@ -25,7 +25,8 @@ static void addAttributes(TemplateVariable template, Variable variable) { if (StringUtils.isNotNullAndNotEmpty(standardName)) { variable.addAttribute(new Attribute("standard_name", standardName)); } - variable.addAttribute(new Attribute("_FillValue", TemplateVariable.getFillValue())); + variable.addAttribute(new Attribute("_FillValue", template.getFillValue())); + variable.addAttribute(new Attribute("missing_value", template.getFillValue())); } static Array readTimeArray(String timeVariableName, NetcdfFile reader) throws IOException, InvalidRangeException { diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSource.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSource.java index 077e9bc47..b2e807ce6 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSource.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSource.java @@ -5,7 +5,7 @@ import com.bc.fiduceo.reader.insitu.gruan_uleic.GruanUleicInsituReader; import com.bc.fiduceo.util.JDomUtils; import com.bc.fiduceo.util.NetCDFUtils; -import org.jdom.Element; +import org.jdom2.Element; import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.InvalidRangeException; diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourcePlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourcePlugin.java index 0e7863ae4..59540c8b0 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourcePlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourcePlugin.java @@ -1,13 +1,14 @@ package com.bc.fiduceo.post.plugin.gruan_uleic; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; -import org.jdom.Element; +import org.jdom2.Element; public class AddGruanSourcePlugin implements PostProcessingPlugin { @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { final AddGruanSource.Configuration configuration = AddGruanSource.parseConfiguration(element); return new AddGruanSource(configuration); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/hirs/flag/HirsL1CloudyFlagsPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/hirs/flag/HirsL1CloudyFlagsPlugin.java index 694c53e8d..7e64741ee 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/hirs/flag/HirsL1CloudyFlagsPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/hirs/flag/HirsL1CloudyFlagsPlugin.java @@ -21,9 +21,10 @@ package com.bc.fiduceo.post.plugin.hirs.flag; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; import com.bc.fiduceo.post.util.DistanceToLandMap; -import org.jdom.Element; +import org.jdom2.Element; import java.nio.file.FileSystem; import java.nio.file.Path; @@ -51,7 +52,7 @@ public class HirsL1CloudyFlagsPlugin implements PostProcessingPlugin { private FileSystem fileSystem; @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { if (!getPostProcessingName().equals(element.getName())) { throw new RuntimeException("Illegal XML Element. Tagname '" + getPostProcessingName() + "' expected."); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrum.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrum.java index 0f62a4eda..01de307ac 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrum.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrum.java @@ -26,7 +26,7 @@ import com.bc.fiduceo.reader.iasi.IASI_Reader; import com.bc.fiduceo.util.JDomUtils; import com.bc.fiduceo.util.NetCDFUtils; -import org.jdom.Element; +import org.jdom2.Element; import ucar.ma2.Array; import ucar.ma2.ArrayFloat; import ucar.ma2.DataType; diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumPlugin.java index 779b4fa0c..a888b5eba 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumPlugin.java @@ -21,13 +21,14 @@ package com.bc.fiduceo.post.plugin.iasi; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; -import org.jdom.Element; +import org.jdom2.Element; public class AddIASISpectrumPlugin implements PostProcessingPlugin { @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { final AddIASISpectrum.Configuration configuration = AddIASISpectrum.createConfiguration(element); return new AddIASISpectrum(configuration); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistance.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistance.java index f556e1f74..be91b0915 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistance.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistance.java @@ -4,9 +4,10 @@ import com.bc.fiduceo.post.util.DistanceToLandMap; import com.bc.fiduceo.util.JDomUtils; import com.bc.fiduceo.util.NetCDFUtils; -import org.jdom.Element; +import org.jdom2.Element; import ucar.ma2.Array; import ucar.ma2.DataType; +import ucar.ma2.IndexIterator; import ucar.ma2.InvalidRangeException; import ucar.nc2.Attribute; import ucar.nc2.Dimension; @@ -18,6 +19,8 @@ import java.nio.file.Paths; import java.util.List; +import static com.bc.fiduceo.post.util.PPUtils.convertToFitTheRangeMinus180to180; + class AddLandDistance extends PostProcessing { @@ -54,6 +57,7 @@ protected void compute(NetcdfFile reader, NetcdfFileWriter writer) throws IOExce final Variable lonVariable = NetCDFUtils.getVariable(reader, configuration.lonVariableName); final Variable latVariable = NetCDFUtils.getVariable(reader, configuration.latVariableName); final Array lonArray = NetCDFUtils.readAndScaleIfNecessary(lonVariable); + convertToFitTheRangeMinus180to180(lonArray); final Array latArray = NetCDFUtils.readAndScaleIfNecessary(latVariable); initDistanceToLandMap(); diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistancePlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistancePlugin.java index 75678964e..2ad8f86bc 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistancePlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistancePlugin.java @@ -2,15 +2,16 @@ import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; -import org.jdom.Element; +import org.jdom2.Element; public class AddLandDistancePlugin implements PostProcessingPlugin { static final String POST_PROCESSING_NAME = "add-distance-to-land"; @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { final AddLandDistance.Configuration configuration = AddLandDistance.createConfiguration(element); return new AddLandDistance(configuration); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPlugin.java index 8c1f13527..cfbd44f8e 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPlugin.java @@ -22,9 +22,10 @@ import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Element; +import org.jdom2.Element; // @todo 2 tb/tb add example XML config as comment here 2017-03-20 @@ -368,7 +369,7 @@ private static int getElementValueInt(Element element) { } @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { final Configuration configuration = createConfiguration(element); if (configuration.verify()) { return new NwpPostProcessing(configuration); diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/point_distance/SphericalDistancePlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/point_distance/SphericalDistancePlugin.java index 18b746ba1..2b78edb47 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/point_distance/SphericalDistancePlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/point_distance/SphericalDistancePlugin.java @@ -20,9 +20,10 @@ package com.bc.fiduceo.post.plugin.point_distance; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Element; +import org.jdom2.Element; public class SphericalDistancePlugin implements PostProcessingPlugin { @@ -40,7 +41,7 @@ public class SphericalDistancePlugin implements PostProcessingPlugin { private static final String OFFSET_ATTR_NAME = "offsetAttrName"; @Override - public PostProcessing createPostProcessing(final Element element) { + public PostProcessing createPostProcessing(final Element element, PostProcessingContext context) { if (!getPostProcessingName().equals(element.getName())) { throw new RuntimeException("Illegal XML Element. Tagname '" + getPostProcessingName() + "' expected."); } diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeries.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeries.java index 674f90443..b731cd2ee 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeries.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeries.java @@ -22,7 +22,7 @@ import com.bc.fiduceo.FiduceoConstants; import com.bc.fiduceo.post.PostProcessing; import com.bc.fiduceo.reader.Reader; -import com.bc.fiduceo.reader.insitu.sst_cci.SSTInsituReader; +import com.bc.fiduceo.reader.insitu.InsituReader; import com.bc.fiduceo.util.NetCDFUtils; import org.esa.snap.core.util.StringUtils; import ucar.ma2.Array; @@ -39,9 +39,10 @@ import static com.bc.fiduceo.util.NetCDFUtils.CF_UNITS_NAME; import static com.bc.fiduceo.util.TimeUtils.secondsSince1978; -class SstInsituTimeSeries extends PostProcessing { +public class SstInsituTimeSeries extends PostProcessing { private static final String FILE_NAME_PATTERN_D8_D8_NC = ".*_\\d{8}_\\d{8}.nc"; + private static final String FILE_NAME_PATTERN_SIRDS = "SSTCCI2_refdata_\\w*_\\d{6}.nc"; static final String INSITU_NTIME = "insitu.ntime"; private final Configuration configuration; @@ -51,7 +52,7 @@ class SstInsituTimeSeries extends PostProcessing { private Variable fileNameVariable; private int filenameFieldSize; - SstInsituTimeSeries(Configuration configuration) { + public SstInsituTimeSeries(Configuration configuration) { this.configuration = configuration; } @@ -65,7 +66,11 @@ protected void prepare(NetcdfFile reader, NetcdfFileWriter writer) throws IOExce String matchupDimensionName = getMatchupDimensionName(); matchupCount = NetCDFUtils.getDimensionLength(matchupDimensionName, reader); - final String insituFileName = getSourceFileName(fileNameVariable, 0, filenameFieldSize, FILE_NAME_PATTERN_D8_D8_NC); + + final String insituFileName = getSourceFileName(fileNameVariable, 0, filenameFieldSize, null); + if (!nameMatches(insituFileName)) { + throw new IOException("The file name '" + insituFileName + "' does not match the regular expressions."); + } final Reader insituReader = readerCache.getReaderFor(sensorType, Paths.get(insituFileName), configuration.processingVersion); addInsituVariables(writer, insituReader); @@ -90,8 +95,11 @@ protected void compute(NetcdfFile reader, NetcdfFileWriter writer) throws IOExce final Variable dtimeVar2D = writer.findVariable(NetCDFUtils.escapeVariableName("insitu.dtime")); for (int i = 0; i < matchupCount; i++) { - final String insituFileName = getSourceFileName(fileNameVariable, i, filenameFieldSize, FILE_NAME_PATTERN_D8_D8_NC); - final SSTInsituReader insituReader = (SSTInsituReader) readerCache.getReaderFor(sensorType, Paths.get(insituFileName), configuration.processingVersion); + final String insituFileName = getSourceFileName(fileNameVariable, i, filenameFieldSize, null); + if (!nameMatches(insituFileName)) { + throw new RuntimeException("File name '" + insituFileName + "' does not match the regular expressions."); + } + final InsituReader insituReader = (InsituReader) readerCache.getReaderFor(sensorType, Paths.get(insituFileName), configuration.processingVersion); Range range = computeInsituRange(y1D[i], insituReader); final int[] origin1D = {range.min}; final int timeSeriesLength = getTimeSeriesLength(range); @@ -156,9 +164,8 @@ static Variable getInsitu_Y_Variable(NetcdfFile reader, final String sensorType, return NetCDFUtils.getVariable(reader, configuration.yVariableName); } - Range computeInsituRange(final int matchupPos, SSTInsituReader insituReader) { - final String name = "insitu.time"; - final Array sourceArray = insituReader.getSourceArray(name); + Range computeInsituRange(final int matchupPos, InsituReader insituReader) throws IOException { + final Array sourceArray = insituReader.getSourceArray(configuration.insituTimeVarName); final int[] times = (int[]) sourceArray.getStorage(); final int matchupTime = times[matchupPos]; final int minTime_ = matchupTime - (configuration.timeRangeSeconds / 2); @@ -232,6 +239,11 @@ private void addAdditionalVariables(NetcdfFileWriter writer, String dimString) { dtimeVariable.addAttribute(new Attribute(CF_FILL_VALUE_NAME, NetCDFUtils.getDefaultFillValue(int.class))); } + // package access for testing only tb 2025-06-27 + static boolean nameMatches(String fileName) { + return fileName.matches(FILE_NAME_PATTERN_D8_D8_NC) || fileName.matches(FILE_NAME_PATTERN_SIRDS); + } + static class Range { final int min; @@ -248,6 +260,7 @@ static class Configuration { int timeRangeSeconds; int timeSeriesSize; String matchupTimeVarName; + String insituTimeVarName; String insituSensorName; String fileNameVariableName; String yVariableName; diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesPlugin.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesPlugin.java index 701ea4526..ffe74f7be 100644 --- a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesPlugin.java +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesPlugin.java @@ -20,9 +20,10 @@ package com.bc.fiduceo.post.plugin.sstInsitu; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; import com.bc.fiduceo.util.JDomUtils; -import org.jdom.Element; +import org.jdom2.Element; public class SstInsituTimeSeriesPlugin implements PostProcessingPlugin { @@ -33,10 +34,11 @@ public class SstInsituTimeSeriesPlugin implements PostProcessingPlugin { static final String TAG_NAME_INSITU_SENSOR = "insitu-sensor"; static final String TAG_NAME_FILE_NAME_VARIABLE = "file-name-variable"; static final String TAG_NAME_Y_VARIABLE = "y-variable"; + static final String TAG_NAME_INSITU_TIME_VARIABLE = "insitu-matchup-time-variable"; static final String TAG_NAME_SECONDARY_SENSOR_MATCHUP_TIME_VARIABLE = "secondary-sensor-matchup-time-variable"; @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { final SstInsituTimeSeries.Configuration configuration = parseConfiguration(element); return new SstInsituTimeSeries(configuration); @@ -62,6 +64,7 @@ static SstInsituTimeSeries.Configuration parseConfiguration(Element element) { final String seriesSize = JDomUtils.getMandatoryChildMandatoryTextTrim(element, TAG_NAME_TIME_SERIES_SIZE); configuration.timeSeriesSize = Integer.parseInt(seriesSize); + configuration.insituTimeVarName = JDomUtils.getMandatoryChildMandatoryTextTrim(element, TAG_NAME_INSITU_TIME_VARIABLE); configuration.matchupTimeVarName = JDomUtils.getMandatoryChildMandatoryTextTrim(element, TAG_NAME_SECONDARY_SENSOR_MATCHUP_TIME_VARIABLE); final Element insituSensorElement = element.getChild(TAG_NAME_INSITU_SENSOR); diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/util/PPUtils.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/util/PPUtils.java new file mode 100644 index 000000000..071d281cf --- /dev/null +++ b/post-processing-tool/src/main/java/com/bc/fiduceo/post/util/PPUtils.java @@ -0,0 +1,24 @@ +package com.bc.fiduceo.post.util; + +import ucar.ma2.Array; +import ucar.ma2.IndexIterator; + +public class PPUtils { + + public static void convertToFitTheRangeMinus180to180(Array lonArray) { + final IndexIterator indexIterator = lonArray.getIndexIterator(); + while (indexIterator.hasNext()) { + double lonD = indexIterator.getDoubleNext(); + if (lonD < 9999 && lonD > -9999 && Double.isFinite(lonD)) { + while (lonD > 180) { + lonD -= 360; + } + while (lonD < -180) { + lonD += 360; + } + indexIterator.setDoubleCurrent(lonD); + } + } + } + +} diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingConfigTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingConfigTest.java index 071ba324b..24161e98e 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingConfigTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingConfigTest.java @@ -20,10 +20,10 @@ package com.bc.fiduceo.post; import com.bc.fiduceo.post.plugin.DummyPostProcessingPlugin; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.output.Format; -import org.jdom.output.XMLOutputter; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.output.Format; +import org.jdom2.output.XMLOutputter; import org.junit.Before; import org.junit.Test; @@ -210,7 +210,7 @@ public void testLoad_CauseIsJDOMParseException() { PostProcessingConfig.load(inputStream); fail("RuntimeException expected"); } catch (RuntimeException expected) { - assertEquals("org.jdom.input.JDOMParseException", expected.getCause().getClass().getTypeName()); + assertEquals("org.jdom2.input.JDOMParseException", expected.getCause().getClass().getTypeName()); assertTrue(expected.getMessage().matches("Unable to initialize post processing configuration: .*")); } } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingFactoryTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingFactoryTest.java index 17a932587..52c2c1d65 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingFactoryTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingFactoryTest.java @@ -19,7 +19,7 @@ package com.bc.fiduceo.post; -import org.jdom.Element; +import org.jdom2.Element; import org.junit.Before; import org.junit.Test; @@ -31,6 +31,7 @@ import static com.bc.fiduceo.post.plugin.point_distance.SphericalDistancePlugin.TAG_NAME_VAR_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -96,8 +97,10 @@ public void testGetPostProcessing() { new Element("secondary-lon-variable").addContent("s_lon") )); - final PostProcessing postProcessing = postProcessingFactory.getPostProcessing(element); + final PostProcessingContext context = new PostProcessingContext(); + final PostProcessing postProcessing = postProcessingFactory.getPostProcessing(element, context); assertNotNull(postProcessing); + assertSame(context, postProcessing.getContext()); assertEquals("com.bc.fiduceo.post.plugin.point_distance.SphericalDistance", postProcessing.getClass().getName()); } @@ -106,7 +109,7 @@ public void testGetPostProcessing_nonExistingPostProcessing() { final Element element = new Element("non-existing-post-processing"); try { - postProcessingFactory.getPostProcessing(element); + postProcessingFactory.getPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("PostProcessing for name 'non-existing-post-processing' not available.", expected.getMessage()); diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolIntegrationTest_Era5.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolIntegrationTest_Era5.java index 6fb786c79..7bb4f3328 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolIntegrationTest_Era5.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolIntegrationTest_Era5.java @@ -191,7 +191,7 @@ public void testAddEra5Variables_coo1() throws IOException, InvalidRangeExceptio NCTestUtils.assertAttribute(variable, "units", "kg kg**-1"); NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 0, 0, 1.9407424645123683E-7, mmd); NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 10, 0, 3.718567541000084E-6, mmd); - NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 20, 0, 9.952551408787258E-6, mmd); + NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 20, 0, 9.968232006940525E-6, mmd); variable = NCTestUtils.getVariable("nwp_u10", mmd); assertNull(variable.findAttribute("standard_name")); @@ -245,8 +245,8 @@ public void testAddEra5Variables_coo1_overwrite() throws IOException, InvalidRan variable = NCTestUtils.getVariable("nwp_o3", mmd); NCTestUtils.assertAttribute(variable, "units", "kg kg**-1"); NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 30, 0, 1.5600191545672715E-5, mmd); - NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 40, 0, 7.585892490169499E-6, mmd); - NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 50, 0, 1.5478117347811349E-6, mmd); + NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 40, 0, 7.563196959381457E-6, mmd); + NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 50, 0, 1.5587879715894815E-6, mmd); } } @@ -286,7 +286,7 @@ public void testAddEra5Variables_coo1_using_sensorRef() throws IOException, Inva NCTestUtils.assertAttribute(variable, "units", "kg kg**-1"); NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 0, 0, 1.9407424645123683E-7, mmd); NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 10, 0, 3.718567541000084E-6, mmd); - NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 20, 0, 9.952551408787258E-6, mmd); + NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 20, 0, 9.968232006940525E-6, mmd); variable = NCTestUtils.getVariable("nwp_u10", mmd); assertNull(variable.findAttribute("standard_name")); @@ -327,6 +327,12 @@ private void writeConfiguration_mmd15() throws IOException { " amsre.acquisition_time" + " amsre.longitude" + " amsre.latitude" + + " nwp_lnsp" + + " nwp_q" + + " nwp_t" + + " nwp_o3" + + " nwp_msl" + + " nwp_tcc" + " " + " " + " 41" + @@ -372,7 +378,7 @@ private void writeConfiguration_coo1(boolean overwrite) throws IOException { configBuffer.append(" "); configBuffer.append(" "); configBuffer.append(" "); - configBuffer.append(" "); + configBuffer.append(" "); configBuffer.append(" slstr.s3ant_nwp_time"); configBuffer.append(" slstr-s3a-nt_acquisition_time"); configBuffer.append(" slstr-s3a-nt_longitude_tx"); @@ -409,12 +415,18 @@ private void writeConfiguration_coo1_with_sensorRef() throws IOException { " slstr-s3a-nt" + " " + " " + - " " + + " " + " {sensor-ref}_nwp_time" + " {sensor-ref}_acquisition_time" + " {sensor-ref}_longitude_tx" + " {sensor-ref}_latitude_tx" + " {sensor-ref}.blowVert" + + " nwp_lnsp" + + " nwp_q" + + " nwp_t" + + " nwp_o3" + + " nwp_msl" + + " nwp_tcc" + " " + " \n" + " \n" + diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolMain_IO_Test.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolMain_IO_Test.java index 76cd6bc8c..c029bdc71 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolMain_IO_Test.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolMain_IO_Test.java @@ -23,10 +23,10 @@ import com.bc.fiduceo.IOTestRunner; import com.bc.fiduceo.TestUtil; import org.esa.snap.core.util.io.FileUtils; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.output.Format; -import org.jdom.output.XMLOutputter; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.output.Format; +import org.jdom2.output.XMLOutputter; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolTest.java index 6ccf65b06..045ba59ee 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolTest.java @@ -24,10 +24,10 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.output.Format; -import org.jdom.output.XMLOutputter; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.output.Format; +import org.jdom2.output.XMLOutputter; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -127,7 +127,7 @@ public void testPrintUsage() { PostProcessingTool.printUsageTo(out); final String ls = System.lineSeparator(); - final String expected = "post-processing-tool version 1.5.8" + ls + + final String expected = "post-processing-tool version 1.6.2" + ls + "" + ls + "usage: post-processing-tool " + ls + "Valid options are:" + ls + diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingTool_IOTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingTool_IOTest.java index c49fad003..645e0a454 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingTool_IOTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingTool_IOTest.java @@ -24,11 +24,11 @@ import com.bc.fiduceo.TestUtil; import com.bc.fiduceo.core.SystemConfig; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.Options; -import org.apache.commons.cli.PosixParser; import org.esa.snap.core.datamodel.ProductData; import org.esa.snap.core.util.io.FileUtils; -import org.jdom.Element; +import org.jdom2.Element; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -83,7 +83,7 @@ public void setUp() throws Exception { } @After - public void tearDown() { + public void tearDown() { if (testDir.isDirectory()) { FileUtils.deleteTree(testDir); } @@ -92,7 +92,7 @@ public void tearDown() { @Test public void testInitialisation() throws Exception { final Options options = PostProcessingTool.getOptions(); - final PosixParser parser = new PosixParser(); + final DefaultParser parser = new DefaultParser(); final CommandLine commandLine = parser.parse(options, new String[]{ "-j", processingConfigName, "-i", "/mmd_files", @@ -105,6 +105,9 @@ public void testInitialisation() throws Exception { final PostProcessingContext context = PostProcessingTool.initializeContext(commandLine); + assertNotNull(context.getConfigDirectory()); + assertEquals(testDir.toPath().resolve("config"), context.getConfigDirectory()); + final String separator = FileSystems.getDefault().getSeparator(); assertEquals(separator + "mmd_files", context.getMmdInputDirectory().toString()); assertEquals("03-May-2011 00:00:00", ProductData.UTC.createDateFormat().format(context.getStartDate())); diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/SourceTargetManagerTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/SourceTargetManagerTest.java index bec3400bc..4baa8e9d6 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/SourceTargetManagerTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/SourceTargetManagerTest.java @@ -22,10 +22,10 @@ import com.bc.fiduceo.TestUtil; import com.bc.fiduceo.post.plugin.DummyPostProcessingPlugin; import org.esa.snap.core.util.io.FileUtils; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.output.Format; -import org.jdom.output.XMLOutputter; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.output.Format; +import org.jdom2.output.XMLOutputter; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/DummyPostProcessingPlugin.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/DummyPostProcessingPlugin.java index e225bd45a..28e80279c 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/DummyPostProcessingPlugin.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/DummyPostProcessingPlugin.java @@ -19,15 +19,16 @@ package com.bc.fiduceo.post.plugin; import com.bc.fiduceo.post.PostProcessing; +import com.bc.fiduceo.post.PostProcessingContext; import com.bc.fiduceo.post.PostProcessingPlugin; -import org.jdom.Element; +import org.jdom2.Element; public class DummyPostProcessingPlugin implements PostProcessingPlugin { public static final String DUMMY_POST_PROCESSING_NAME = "dummy-post-processing"; @Override - public PostProcessing createPostProcessing(Element element) { + public PostProcessing createPostProcessing(Element element, PostProcessingContext context) { return new DummyPostProcessing(element.getValue()); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/ElevationToSolZenAnglePluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/ElevationToSolZenAnglePluginTest.java index c18b098af..011352fb7 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/ElevationToSolZenAnglePluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/ElevationToSolZenAnglePluginTest.java @@ -3,8 +3,8 @@ import com.bc.fiduceo.TestUtil; import com.bc.fiduceo.post.PostProcessing; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; @@ -37,7 +37,7 @@ public void testGetPostProcessingName() { public void testCreatePostProcessing() throws JDOMException, IOException { final Element rootElement = TestUtil.createDomElement(FULL_CONFIG); - final PostProcessing postProcessing = plugin.createPostProcessing(rootElement); + final PostProcessing postProcessing = plugin.createPostProcessing(rootElement, null); assertNotNull(postProcessing); assertTrue(postProcessing instanceof ElevationToSolZenAngle); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/amsr/AddAmsr2ScanDataQualityPluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/amsr/AddAmsr2ScanDataQualityPluginTest.java index c940d1708..d7bfa8af7 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/amsr/AddAmsr2ScanDataQualityPluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/amsr/AddAmsr2ScanDataQualityPluginTest.java @@ -25,10 +25,8 @@ import com.bc.fiduceo.TestUtil; import com.bc.fiduceo.post.PostProcessing; -import com.bc.fiduceo.post.plugin.amsr.AddAmsr2ScanDataQuality; -import com.bc.fiduceo.post.plugin.amsr.AddAmsr2ScanDataQualityPlugin; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.*; import java.io.IOException; @@ -58,7 +56,7 @@ public void testGetPostProcessingName() { public void testCreatePostProcessing() throws JDOMException, IOException { final Element rootElement = TestUtil.createDomElement(FULL_CONFIG); - final PostProcessing postProcessing = plugin.createPostProcessing(rootElement); + final PostProcessing postProcessing = plugin.createPostProcessing(rootElement, null); assertNotNull(postProcessing); assertTrue(postProcessing instanceof AddAmsr2ScanDataQuality); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/amsr/AddAmsrSolarAnglesPluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/amsr/AddAmsrSolarAnglesPluginTest.java index 8dfa22c7e..e05d5533f 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/amsr/AddAmsrSolarAnglesPluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/amsr/AddAmsrSolarAnglesPluginTest.java @@ -26,10 +26,8 @@ import com.bc.fiduceo.TestUtil; import com.bc.fiduceo.post.PostProcessing; -import com.bc.fiduceo.post.plugin.amsr.AddAmsrSolarAngles; -import com.bc.fiduceo.post.plugin.amsr.AddAmsrSolarAnglesPlugin; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.*; import java.io.IOException; @@ -61,7 +59,7 @@ public void testGetPostProcessingName() { public void testCreatePostProcessing() throws JDOMException, IOException { final Element rootElement = TestUtil.createDomElement(FULL_CONFIG); - final PostProcessing postProcessing = plugin.createPostProcessing(rootElement); + final PostProcessing postProcessing = plugin.createPostProcessing(rootElement, null); assertNotNull(postProcessing); assertTrue(postProcessing instanceof AddAmsrSolarAngles); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsPluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsPluginTest.java index abec91598..91c3bd1d3 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsPluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsPluginTest.java @@ -2,8 +2,8 @@ import com.bc.fiduceo.TestUtil; import com.bc.fiduceo.post.PostProcessing; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; @@ -36,7 +36,7 @@ public void testCreatePostProcessing() throws JDOMException, IOException { " " + ""); - final PostProcessing postProcessing = plugin.createPostProcessing(element); + final PostProcessing postProcessing = plugin.createPostProcessing(element, null); assertNotNull(postProcessing); assertTrue(postProcessing instanceof AddAvhrrCorrCoeffs); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsTest.java index 8345fdca1..e0b8d12fe 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffsTest.java @@ -1,8 +1,8 @@ package com.bc.fiduceo.post.plugin.avhrr_fcdr; import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Test; import java.io.IOException; diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/caliop/flag/CALIOP_L2_VFM_FLAGS_PPPluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/caliop/flag/CALIOP_L2_VFM_FLAGS_PPPluginTest.java index 0ff337863..3ee456342 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/caliop/flag/CALIOP_L2_VFM_FLAGS_PPPluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/caliop/flag/CALIOP_L2_VFM_FLAGS_PPPluginTest.java @@ -21,7 +21,7 @@ package com.bc.fiduceo.post.plugin.caliop.flag; import com.bc.fiduceo.post.PostProcessing; -import org.jdom.Element; +import org.jdom2.Element; import org.junit.Before; import org.junit.Test; @@ -53,7 +53,7 @@ public void getPostProcessingName() { @Test public void createPostProcessing() { - final PostProcessing pp = ppPlugin.createPostProcessing(createValidRootElement()); + final PostProcessing pp = ppPlugin.createPostProcessing(createValidRootElement(), null); assertNotNull(pp); assertEquals(CALIOP_L2_VFM_FLAGS_PP.class, pp.getClass()); final CALIOP_L2_VFM_FLAGS_PP cfpp = (CALIOP_L2_VFM_FLAGS_PP) pp; @@ -67,7 +67,7 @@ public void createPostProcessing() { public void createPostProcessing_wrongRootTag() { final Element rootElement = createValidRootElement(); try { - ppPlugin.createPostProcessing(rootElement.setName("wrongRootName")); + ppPlugin.createPostProcessing(rootElement.setName("wrongRootName"), null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -80,7 +80,7 @@ public void createPostProcessing_sourceFileVarName_missingElement() { final Element rootElement = createValidRootElement(); rootElement.removeChild(TAG_MMD_SOURCE_FILE_VARIABE_NAME); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -93,7 +93,7 @@ public void createPostProcessing_sourceFileVarName_empty() { final Element rootElement = createValidRootElement(); rootElement.getChild(TAG_MMD_SOURCE_FILE_VARIABE_NAME).setText(" "); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -107,7 +107,7 @@ public void createPostProcessing_processingVersionVarName_missingElement() { final Element rootElement = createValidRootElement(); rootElement.removeChild(TAG_MMD_PROCESSING_VERSION_VARIABE_NAME); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -120,7 +120,7 @@ public void createPostProcessing_processingVersionVarName_empty() { final Element rootElement = createValidRootElement(); rootElement.getChild(TAG_MMD_PROCESSING_VERSION_VARIABE_NAME).setText(" "); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -133,7 +133,7 @@ public void createPostProcessing_yVarName_missingElement() { final Element rootElement = createValidRootElement(); rootElement.removeChild(TAG_MMD_Y_VARIABE_NAME); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -146,7 +146,7 @@ public void createPostProcessing_yVarName_empty() { final Element rootElement = createValidRootElement(); rootElement.getChild(TAG_MMD_Y_VARIABE_NAME).setText(" "); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -159,7 +159,7 @@ public void createPostProcessing_targetVarName_missingElement() { final Element rootElement = createValidRootElement(); rootElement.removeChild(TAG_TARGET_FCF_VARIABLE_NAME); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -172,7 +172,7 @@ public void createPostProcessing_targetVarName_empty() { final Element rootElement = createValidRootElement(); rootElement.getChild(TAG_TARGET_FCF_VARIABLE_NAME).setText(" "); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/caliop/sst_wp100/CALIOP_SST_WP100_CLay_PPPluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/caliop/sst_wp100/CALIOP_SST_WP100_CLay_PPPluginTest.java index f405e1d11..94cc4a8d6 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/caliop/sst_wp100/CALIOP_SST_WP100_CLay_PPPluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/caliop/sst_wp100/CALIOP_SST_WP100_CLay_PPPluginTest.java @@ -21,7 +21,7 @@ package com.bc.fiduceo.post.plugin.caliop.sst_wp100; import com.bc.fiduceo.post.PostProcessing; -import org.jdom.Element; +import org.jdom2.Element; import org.junit.Before; import org.junit.Test; @@ -53,7 +53,7 @@ public void getPostProcessingName() { @Test public void createPostProcessing() { - final PostProcessing pp = ppPlugin.createPostProcessing(createValidRootElement()); + final PostProcessing pp = ppPlugin.createPostProcessing(createValidRootElement(), null); assertNotNull(pp); assertEquals(CALIOP_SST_WP100_CLay_PP.class, pp.getClass()); final CALIOP_SST_WP100_CLay_PP cwp100_CLay_pp = (CALIOP_SST_WP100_CLay_PP) pp; @@ -67,7 +67,7 @@ public void createPostProcessing() { public void createPostProcessing_wrongRootTag() { final Element rootElement = createValidRootElement(); try { - ppPlugin.createPostProcessing(rootElement.setName("wrongRootName")); + ppPlugin.createPostProcessing(rootElement.setName("wrongRootName"), null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -80,7 +80,7 @@ public void createPostProcessing_sourceFileVarName_missingElement() { final Element rootElement = createValidRootElement(); rootElement.removeChild(TAG_MMD_SOURCE_FILE_VARIABE_NAME); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -93,7 +93,7 @@ public void createPostProcessing_sourceFileVarName_empty() { final Element rootElement = createValidRootElement(); rootElement.getChild(TAG_MMD_SOURCE_FILE_VARIABE_NAME).setText(" "); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -107,7 +107,7 @@ public void createPostProcessing_processingVersionVarName_missingElement() { final Element rootElement = createValidRootElement(); rootElement.removeChild(TAG_MMD_PROCESSING_VERSION); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -120,7 +120,7 @@ public void createPostProcessing_processingVersionVarName_empty() { final Element rootElement = createValidRootElement(); rootElement.getChild(TAG_MMD_PROCESSING_VERSION).setText(" "); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -133,7 +133,7 @@ public void createPostProcessing_yVarName_missingElement() { final Element rootElement = createValidRootElement(); rootElement.removeChild(TAG_MMD_Y_VARIABE_NAME); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -146,7 +146,7 @@ public void createPostProcessing_yVarName_empty() { final Element rootElement = createValidRootElement(); rootElement.getChild(TAG_MMD_Y_VARIABE_NAME).setText(" "); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -159,7 +159,7 @@ public void createPostProcessing_targetPrefix_missingElement() { final Element rootElement = createValidRootElement(); rootElement.removeChild(TAG_TARGET_VARIABE_PREFIX); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); @@ -172,7 +172,7 @@ public void createPostProcessing_targetPrefix_empty() { final Element rootElement = createValidRootElement(); rootElement.getChild(TAG_TARGET_VARIABE_PREFIX).setText(" "); try { - ppPlugin.createPostProcessing(rootElement); + ppPlugin.createPostProcessing(rootElement, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals(RuntimeException.class.getTypeName(), expected.getClass().getTypeName()); diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/ArrayUtilsTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/ArrayUtilsTest.java new file mode 100644 index 000000000..d63f8512d --- /dev/null +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/ArrayUtilsTest.java @@ -0,0 +1,142 @@ +package com.bc.fiduceo.post.plugin.era5; + +import org.junit.Test; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.Index; + +import static org.junit.Assert.*; + +public class ArrayUtilsTest { + + @Test + public void testMergeALongX_3D() { + final int[] dataLeft = new int[]{0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + // ------------ + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23}; + final int[] shapeLeft = new int[]{2, 3, 4}; + + final Array left = Array.factory(DataType.INT, shapeLeft, dataLeft); + + final int[] dataRight = new int[]{100, 101, 102, + 104, 105, 106, + 108, 109, 110, + // ---------- + 112, 113, 114, + 116, 117, 118, + 120, 121, 122}; + final int[] shapeRight = new int[]{2, 3, 3}; + final Array right = Array.factory(DataType.INT, shapeRight, dataRight); + + final Array merged = ArrayUtils.mergeAlongX(left, right); + int[] shape = merged.getShape(); + assertArrayEquals(new int[]{2, 3, 7}, shape); + + final Index index = merged.getIndex(); + index.set(0, 0, 0); + assertEquals(0, merged.getInt(index)); + index.set(0, 0, 3); + assertEquals(3, merged.getInt(index)); + index.set(0, 0, 4); + assertEquals(100, merged.getInt(index)); + index.set(0, 0, 6); + assertEquals(102, merged.getInt(index)); + + index.set(1, 2, 0); + assertEquals(20, merged.getInt(index)); + index.set(1, 2, 3); + assertEquals(23, merged.getInt(index)); + index.set(1, 2, 4); + assertEquals(120, merged.getInt(index)); + index.set(1, 2, 6); + assertEquals(122, merged.getInt(index)); + } + + @Test + public void testMergeALongX_4D() { + final int[] dataLeft = new int[]{0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + // ------------ + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23, + // ============ + 30, 31, 32, 33, + 34, 35, 36, 37, + 38, 39, 40, 41, + // ------------ + 42, 43, 44, 45, + 46, 47, 48, 49, + 50, 51, 52, 53}; + final int[] shapeLeft = new int[]{2, 2, 3, 4}; + + final Array left = Array.factory(DataType.INT, shapeLeft, dataLeft); + + final int[] dataRight = new int[]{100, 101, 102, + 104, 105, 106, + 108, 109, 110, + // ---------- + 112, 113, 114, + 116, 117, 118, + 120, 121, 122, + // ========== + 200, 201, 202, + 204, 205, 206, + 208, 209, 210, + // ---------- + 212, 213, 214, + 216, 217, 218, + 220, 221, 222}; + final int[] shapeRight = new int[]{2, 2, 3, 3}; + final Array right = Array.factory(DataType.INT, shapeRight, dataRight); + + final Array merged = ArrayUtils.mergeAlongX(left, right); + int[] shape = merged.getShape(); + assertArrayEquals(new int[]{2, 2, 3, 7}, shape); + + final Index index = merged.getIndex(); + index.set(0, 0, 0, 0); + assertEquals(0, merged.getInt(index)); + index.set(0, 0, 0, 3); + assertEquals(3, merged.getInt(index)); + index.set(0, 0, 0, 4); + assertEquals(100, merged.getInt(index)); + index.set(0, 0, 0, 6); + assertEquals(102, merged.getInt(index)); + + index.set(1, 1, 2, 0); + assertEquals(50, merged.getInt(index)); + index.set(1, 1, 2, 3); + assertEquals(53, merged.getInt(index)); + index.set(1, 1, 2, 4); + assertEquals(220, merged.getInt(index)); + index.set(1, 1, 2, 6); + assertEquals(222, merged.getInt(index)); + } + + @Test + public void testMergeALongX_invalidDimension() { + Array left = Array.factory(DataType.FLOAT, new int[]{4, 6}); + Array right = Array.factory(DataType.FLOAT, new int[]{4, 6}); + + try { + ArrayUtils.mergeAlongX(left, right); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException expected) { + } + + left = Array.factory(DataType.FLOAT, new int[]{2, 2, 2, 4, 6}); + right = Array.factory(DataType.FLOAT, new int[]{2, 2, 2, 4, 6}); + + try { + ArrayUtils.mergeAlongX(left, right); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException expected) { + } + } +} diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/BilinearInterpolatorTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/BilinearInterpolatorTest.java index c44dfd85c..51511c826 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/BilinearInterpolatorTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/BilinearInterpolatorTest.java @@ -82,5 +82,23 @@ public void testConstructAndGetCoordinates() { assertEquals(62, interpolator.getXMin()); assertEquals(129, interpolator.getYMin()); + assertEquals(Integer.MIN_VALUE, interpolator.getRelXMin()); + assertEquals(Integer.MIN_VALUE, interpolator.getRelYMin()); + } + + @Test + public void testSetGetRelXMin() { + final BilinearInterpolator interpolator = new BilinearInterpolator(0.0, 0.0, 63, 130); + + interpolator.setRelXMin(5); + assertEquals(5, interpolator.getRelXMin()); + } + + @Test + public void testSetGetRelYMin() { + final BilinearInterpolator interpolator = new BilinearInterpolator(0.0, 0.0, 63, 130); + + interpolator.setRelYMin(6); + assertEquals(6, interpolator.getRelYMin()); } } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5ArchiveTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5ArchiveTest.java index 4f20da184..6eeabcf11 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5ArchiveTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5ArchiveTest.java @@ -1,11 +1,10 @@ package com.bc.fiduceo.post.plugin.era5; import com.bc.fiduceo.util.TimeUtils; -import org.junit.Ignore; +import org.junit.Before; import org.junit.Test; import java.io.File; -import java.io.IOException; import java.util.Calendar; import static junit.framework.TestCase.assertEquals; @@ -14,10 +13,18 @@ public class Era5ArchiveTest { private static final String SEP = File.separator; + private Configuration config; + + @Before + public void setUp() throws Exception { + config = new Configuration(); + } @Test - public void testConstructAndGet() { - final Era5Archive era5Archive = new Era5Archive("archive" + SEP + "era5", Era5Collection.ERA_5); + public void testConstructAndGet_translateVariableNameToFileAccessName_true() { + config.setNWPAuxDir("archive" + SEP + "era5"); + config.setTranslateVariableNameToFileAccessName(true); // default is true + final Era5Archive era5Archive = new Era5Archive(config, Era5Collection.ERA_5); // Friday, 30. May 2008 11:00:00 // 1212145200 @@ -39,21 +46,50 @@ public void testConstructAndGet() { assertEquals(expected, era5Archive.get("fc_sfc_mslhf", 1212400800)); } + @Test + public void testConstructAndGet_translateVariableNameToFileAccessName_false() { + config.setNWPAuxDir("archive" + SEP + "era5"); + config.setTranslateVariableNameToFileAccessName(false); + final Era5Archive era5Archive = new Era5Archive(config, Era5Collection.ERA_5); + + // Friday, 30. May 2008 11:00:00 + // 1212145200 + String expected = assemblePath("archive", "era5", "an_ml", "2008", "05", "30", "ecmwf-era5_oper_an_ml_200805301100.q.nc"); + assertEquals(expected, era5Archive.get("an_ml_q", 1212145200)); + + expected = assemblePath("archive", "era5", "an_sfc", "2008", "05", "30", "ecmwf-era5_oper_an_sfc_200805301100.t2m.nc"); + assertEquals(expected, era5Archive.get("an_sfc_t2m", 1212145200)); + + expected = assemblePath("archive", "era5", "fc_sfc", "2008", "05", "30", "ecmwf-era5_oper_fc_sfc_2008053006005.msnlwrf.nc"); + assertEquals(expected, era5Archive.get("fc_sfc_msnlwrf", 1212145200)); + + // Monday, 2. June 2008 10:00:00 + // 1212400800 + expected = assemblePath("archive", "era5", "an_ml", "2008", "06", "02", "ecmwf-era5_oper_an_ml_200806021000.t.nc"); + assertEquals(expected, era5Archive.get("an_ml_t", 1212400800)); + + expected = assemblePath("archive", "era5", "fc_sfc", "2008", "06", "02", "ecmwf-era5_oper_fc_sfc_2008060206004.mslhf.nc"); + assertEquals(expected, era5Archive.get("fc_sfc_mslhf", 1212400800)); + } + @Test public void testGetFileName_era5() { - final Era5Archive archive = new Era5Archive("whatever", Era5Collection.ERA_5); + config.setNWPAuxDir("whatever"); + final Era5Archive archive = new Era5Archive(config, Era5Collection.ERA_5); assertEquals("ecmwf-era5_oper_an_ml_201108231900.q.nc", archive.getFileName("an_ml", "q", "201108231900")); } @Test public void testGetFileName_era5t() { - final Era5Archive archive = new Era5Archive("whatever", Era5Collection.ERA_5T); + config.setNWPAuxDir("whatever"); + final Era5Archive archive = new Era5Archive(config, Era5Collection.ERA_5T); assertEquals("ecmwf-era5t_oper_an_ml_201108232000.q.nc", archive.getFileName("an_ml", "q", "201108232000")); } @Test public void testGetFileName_era51() { - final Era5Archive archive = new Era5Archive("whatever", Era5Collection.ERA_51); + config.setNWPAuxDir("whatever"); + final Era5Archive archive = new Era5Archive(config, Era5Collection.ERA_51); assertEquals("ecmwf-era51_oper_an_fc_201108232000.q.nc", archive.getFileName("an_fc", "q", "201108232000")); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPluginTest.java index 257feff6d..f0f9f4a71 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPluginTest.java @@ -2,22 +2,27 @@ import com.bc.fiduceo.TestUtil; import com.bc.fiduceo.post.PostProcessing; -import org.jdom.Element; -import org.jdom.JDOMException; +import com.bc.fiduceo.post.PostProcessingContext; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; import java.io.IOException; +import java.nio.file.Paths; import static org.junit.Assert.*; public class Era5PostProcessingPluginTest { private Era5PostProcessingPlugin plugin; + private PostProcessingContext context; @Before public void setUp() { plugin = new Era5PostProcessingPlugin(); + context = new PostProcessingContext(); + context.setConfigDirectory(Paths.get(".config")); } @Test @@ -32,7 +37,7 @@ public void testCreateConfiguration_missing_nwpAuxDir() throws JDOMException, IO final Element rootElement = TestUtil.createDomElement(XML); try { - Era5PostProcessingPlugin.createConfiguration(rootElement); + Era5PostProcessingPlugin.createConfiguration(rootElement, context); fail("RuntimeException expected"); } catch (RuntimeException expected) { } @@ -45,7 +50,7 @@ public void testCreateConfiguration_nwpAuxDir() throws JDOMException, IOExceptio ""; final Element rootElement = TestUtil.createDomElement(XML); - final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement); + final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement, context); assertEquals("/where/the/data/is", configuration.getNWPAuxDir()); } @@ -80,25 +85,25 @@ public void testCreateConfiguration_satelliteFields() throws JDOMException, IOEx ""; final Element rootElement = TestUtil.createDomElement(XML); - final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement); + final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement, context); assertEquals("The-One", configuration.getEra5Collection()); final SatelliteFieldsConfiguration satConfig = configuration.getSatelliteFields(); assertNotNull(satConfig); - assertEquals("Kjuh", satConfig.get_an_q_name()); - assertEquals("tea", satConfig.get_an_t_name()); - assertEquals("ozone", satConfig.get_an_o3_name()); - assertEquals("pressure", satConfig.get_an_lnsp_name()); - assertEquals("tempi", satConfig.get_an_t2m_name()); - assertEquals("blowUp", satConfig.get_an_u10_name()); - assertEquals("blowVert", satConfig.get_an_v10_name()); - assertEquals("concentrate", satConfig.get_an_siconc_name()); - assertEquals("meanPress", satConfig.get_an_msl_name()); - assertEquals("skinTemp", satConfig.get_an_skt_name()); - assertEquals("ozeanTemp", satConfig.get_an_sst_name()); - assertEquals("cloudy", satConfig.get_an_tcc_name()); - assertEquals("steam!", satConfig.get_an_tcwv_name()); + assertEquals("Kjuh", satConfig.getVarName("an_ml_q")); + assertEquals("tea", satConfig.getVarName("an_ml_t")); + assertEquals("ozone", satConfig.getVarName("an_ml_o3")); + assertEquals("pressure", satConfig.getVarName("an_ml_lnsp")); + assertEquals("tempi", satConfig.getVarName("an_sfc_t2m") ); + assertEquals("blowUp", satConfig.getVarName("an_sfc_u10") ); + assertEquals("blowVert", satConfig.getVarName("an_sfc_v10") ); + assertEquals("concentrate", satConfig.getVarName("an_sfc_siconc") ); + assertEquals("meanPress", satConfig.getVarName("an_sfc_msl")); + assertEquals("skinTemp", satConfig.getVarName("an_sfc_skt")); + assertEquals("ozeanTemp", satConfig.getVarName("an_sfc_sst")); + assertEquals("cloudy", satConfig.getVarName("an_sfc_tcc")); + assertEquals("steam!", satConfig.getVarName("an_sfc_tcwv")); assertEquals(5, satConfig.get_x_dim()); assertEquals("left", satConfig.get_x_dim_name()); @@ -145,25 +150,25 @@ public void testCreateConfiguration_satelliteFields_with_sensorRef() throws JDOM ""; final Element rootElement = TestUtil.createDomElement(XML); - final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement); + final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement, context); assertEquals("The-One", configuration.getEra5Collection()); final SatelliteFieldsConfiguration satConfig = configuration.getSatelliteFields(); assertNotNull(satConfig); - assertEquals("hirs-n08_Kjuh", satConfig.get_an_q_name()); - assertEquals("hirs-n08_tea", satConfig.get_an_t_name()); - assertEquals("hirs-n08_ozone", satConfig.get_an_o3_name()); - assertEquals("hirs-n08_pressure", satConfig.get_an_lnsp_name()); - assertEquals("hirs-n08_tempi", satConfig.get_an_t2m_name()); - assertEquals("hirs-n08_blowUp", satConfig.get_an_u10_name()); - assertEquals("hirs-n08_blowVert", satConfig.get_an_v10_name()); - assertEquals("hirs-n08_concentrate", satConfig.get_an_siconc_name()); - assertEquals("hirs-n08_meanPress", satConfig.get_an_msl_name()); - assertEquals("hirs-n08_skinTemp", satConfig.get_an_skt_name()); - assertEquals("hirs-n08_ozeanTemp", satConfig.get_an_sst_name()); - assertEquals("hirs-n08_cloudy", satConfig.get_an_tcc_name()); - assertEquals("hirs-n08_steam!", satConfig.get_an_tcwv_name()); + // assertEquals("hirs-n08_Kjuh", satConfig.getVarName("an_ml_q")); + // assertEquals("hirs-n08_tea", satConfig.getVarName("an_ml_t")); + // assertEquals("hirs-n08_ozone", satConfig.getVarName("an_ml_o3")); + // assertEquals("hirs-n08_pressure", satConfig.getVarName("an_ml_lnsp")); + assertEquals("hirs-n08_tempi", satConfig.getVarName("an_sfc_t2m") ); + assertEquals("hirs-n08_blowUp", satConfig.getVarName("an_sfc_u10") ); + assertEquals("hirs-n08_blowVert", satConfig.getVarName("an_sfc_v10") ); + assertEquals("hirs-n08_concentrate", satConfig.getVarName("an_sfc_siconc") ); + // assertEquals("hirs-n08_meanPress", satConfig.getVarName("an_sfc_msl")); + assertEquals("hirs-n08_skinTemp", satConfig.getVarName("an_sfc_skt")); + assertEquals("hirs-n08_ozeanTemp", satConfig.getVarName("an_sfc_sst")); + // assertEquals("hirs-n08_cloudy", satConfig.getVarName("an_sfc_tcc")); + assertEquals("hirs-n08_steam!", satConfig.getVarName("an_sfc_tcwv")); assertEquals(5, satConfig.get_x_dim()); assertEquals("left", satConfig.get_x_dim_name()); @@ -178,6 +183,9 @@ public void testCreateConfiguration_satelliteFields_with_sensorRef() throws JDOM assertEquals("sensor_clock", satConfig.get_time_variable_name()); } + // todo sabine .. done 2021-02-20 + // The "length" attribute of tag " is no longer optional. +/* @Test public void testCreateConfiguration_satelliteFields_zDimNotSet() throws JDOMException, IOException { final String XML = "" + @@ -186,7 +194,7 @@ public void testCreateConfiguration_satelliteFields_zDimNotSet() throws JDOMExce " " + " " + " " + - " " + + " " + " era5-time" + " along_way" + " alattemacchiato" + @@ -195,9 +203,10 @@ public void testCreateConfiguration_satelliteFields_zDimNotSet() throws JDOMExce ""; final Element rootElement = TestUtil.createDomElement(XML); - final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement); + final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement, null); assertEquals(137, configuration.getSatelliteFields().get_z_dim()); } +*/ @Test public void testCreateConfiguration_matchupFields() throws JDOMException, IOException { @@ -226,7 +235,7 @@ public void testCreateConfiguration_matchupFields() throws JDOMException, IOExce ""; final Element rootElement = TestUtil.createDomElement(XML); - final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement); + final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement, context); final MatchupFieldsConfiguration matchupConfig = configuration.getMatchupFields(); assertNotNull(matchupConfig); @@ -278,7 +287,7 @@ public void testCreateConfiguration_matchupFields_with_insituRef() throws JDOMEx ""; final Element rootElement = TestUtil.createDomElement(XML); - final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement); + final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement, context); final MatchupFieldsConfiguration matchupConfig = configuration.getMatchupFields(); assertNotNull(matchupConfig); @@ -311,7 +320,7 @@ public void testCreatePostProcessing() throws JDOMException, IOException { ""; final Element rootElement = TestUtil.createDomElement(XML); - final PostProcessing postProcessing = plugin.createPostProcessing(rootElement); + final PostProcessing postProcessing = plugin.createPostProcessing(rootElement, context); assertNotNull(postProcessing); assertTrue(postProcessing instanceof Era5PostProcessing); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPluginTest_parseGeneralizedInformations.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPluginTest_parseGeneralizedInformations.java new file mode 100644 index 000000000..299ee2452 --- /dev/null +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPluginTest_parseGeneralizedInformations.java @@ -0,0 +1,317 @@ +package com.bc.fiduceo.post.plugin.era5; + +import com.bc.fiduceo.TestUtil; +import com.google.common.jimfs.Jimfs; +import org.jdom2.Element; +import org.jdom2.JDOMException; +import org.jdom2.input.JDOMParseException; +import org.jdom2.output.Format; +import org.jdom2.output.XMLOutputter; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.when; + +public class Era5PostProcessingPluginTest_parseGeneralizedInformations { + + @Test + public void testParseGeneralizedInformation_unableToCreateInputStream_mockIOException() { + //preparation + final Path configPathMock = Mockito.mock(Path.class); + final Path infoPathMock = Mockito.mock(Path.class); + when(configPathMock.resolve(Mockito.anyString())).thenReturn(infoPathMock); + when(infoPathMock.toString()).thenReturn("somePathName"); + try (MockedStatic filesMock = Mockito.mockStatic(Files.class)) { + filesMock.when(() -> Files.exists(infoPathMock)).thenReturn(true); + filesMock.when(() -> Files.newInputStream(infoPathMock)).thenThrow(new IOException("mocked")); + + //execution + Era5PostProcessingPlugin.parseGeneralizedInformation(new Configuration(), configPathMock); + + //verification + fail("RuntimeException expected"); + } catch (Throwable e) { + assertThat(e.getClass(), is(equalTo(RuntimeException.class))); + assertEquals("Unable to create input stream from somePathName.", e.getMessage()); + final Throwable cause = e.getCause(); + assertThat(cause.getClass(), is(equalTo(IOException.class))); + assertEquals("mocked", cause.getMessage()); + } + } + + @Test + public void testParseGeneralizedInformation_generalInfoIsNotValidXml() throws IOException { + //preparation + final String XML = "habi dubi \n" + + " babi \n" + + " \n" + + ""; + final Path configPath = Jimfs.newFileSystem().getPath("config"); + + final Path xmlPath = configPath.resolve(Era5PostProcessingPlugin.ERA5_POST_PROCESSING_GENERAL_INFO_XML); + Files.createDirectories(configPath); + try ( + OutputStream os = Files.newOutputStream(xmlPath); + PrintWriter pw = new PrintWriter(os) + ) { + pw.println(XML); + } + + try { + //execution + Era5PostProcessingPlugin.parseGeneralizedInformation(new Configuration(), configPath); + + //verification + fail("RuntimeException expected"); + } catch (Throwable e) { + assertThat(e.getClass(), is(equalTo(RuntimeException.class))); + assertEquals("XML document " + xmlPath + " could not be read in.", e.getMessage()); + final Throwable cause = e.getCause(); + assertThat(cause.getClass(), is(equalTo(JDOMParseException.class))); + assertThat(cause.getMessage(), startsWith("Error on line 1:")); + } + } + + @Test + public void testParseGeneralizedInformation_fromFile_rootTagIsNot_era5() throws IOException, JDOMException { + //preparation + final String XML = "\n" + + " \n" + + " \n" + + ""; + final Path configPath = output(XML); + + try { + //execution + Era5PostProcessingPlugin.parseGeneralizedInformation(new Configuration(), configPath); + + //verification + fail("RuntimeException expected"); + } catch (Throwable e) { + assertThat(e.getClass(), is(equalTo(RuntimeException.class))); + assertEquals("Root tag expected in config"+ File.separator + "era5-post-processing-general-info.xml", e.getMessage()); + final Throwable cause = e.getCause(); + assertThat(cause.getClass(), is(equalTo(Throwable.class))); + assertEquals("Root tag expected", cause.getMessage()); + } + } + + @Test + public void testParseGeneralizedInformation_rootTagIsNot_era5() throws IOException, JDOMException { + //preparation + final String XML = "\n" + + " \n" + + " \n" + + ""; + final Element rootElement = TestUtil.createDomElement(XML); + + try { + //execution + Era5PostProcessingPlugin.parseGeneralizedInformation(rootElement, new Configuration()); + + //verification + fail("Throwable expected"); + } catch (Throwable e) { + assertThat(e.getClass(), is(equalTo(Throwable.class))); + assertEquals("Root tag expected", e.getMessage()); + } + } + + @Test + public void testParseGeneralizedInformation_unknownAttributeName() throws IOException, JDOMException { + //preparation + final String XML = "\n" + + " \n" + + " \n" + + " true\n" + + " \n" + + " K\n" + + " \n" + + " \n" + + " \n" + + ""; + final Element rootElement = TestUtil.createDomElement(XML); + + try { + //execution + Era5PostProcessingPlugin.parseGeneralizedInformation(rootElement, new Configuration()); + + //verification + fail("Throwable expected"); + } catch (Throwable e) { + assertThat(e.getClass(), is(equalTo(Throwable.class))); + assertEquals("Unknown attribute name Pumpernikel", e.getMessage()); + } + } + + @Test + public void testParseGeneralizedInformation_noGenaralInformationFile() throws IOException, JDOMException { + //preparation + final Path configPath = Jimfs.newFileSystem().getPath("config"); + Files.createDirectories(configPath); + final Configuration config = new Configuration(); + + //execution + Era5PostProcessingPlugin.parseGeneralizedInformation(config, configPath); + + //verification + assertThat(config.getSatelliteFields(), is(nullValue())); + } + + @Test + public void testParseGeneralizedInformation_readFromFile() throws IOException, JDOMException { + //preparation + final String XML = "\n" + + " \n" + + " \n" + + " true\n" + + " \n" + + " K\n" + + " Temperature\n" + + " air_temperature\n" + + " 24.3\n" + + " \n" + + " \n" + + " kg kg**-1\n" + + " Specific humidity\n" + + " specific_humidity\n" + + " 24.3e-5\n" + + " \n" + + " \n" + + " kg kg**-1\n" + + " Specific cloud liquid water content\n" + + " \n" + + " \n" + + " kg kg**-1\n" + + " Specific rain water content\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " kg m**-2\n" + + " Total column cloud liquid water\n" + + " \n" + + " \n" + + " kg m**-2\n" + + " Total column cloud ice water\n" + + " \n" + + " \n" + + " kg m**-2\n" + + " Total column rain water\n" + + " \n" + + " \n" + + " kg m**-2\n" + + " Total column snow water\n" + + " \n" + + " \n" + + " \n" + + ""; + final Path configPath = output(XML); + final Configuration configuration = new Configuration(); + + //execution + Era5PostProcessingPlugin.parseGeneralizedInformation(configuration, configPath); + + //verification + final SatelliteFieldsConfiguration satConf = configuration.getSatelliteFields(); + assertNotNull(satConf); + final Map genVars = satConf.getGeneralizedVariables(); + assertNotNull(genVars); + assertThat(genVars.size(), is(8)); + + TemplateVariable var = genVars.get("an_pl_t"); + assertNotNull(var); + assertThat(var.getName(), is("t")); + assertThat(var.getUnits(), is("K")); + assertThat(var.getLongName(), is("Temperature")); + assertThat(var.getStandardName(), is("air_temperature")); + assertThat(var.getFillValue(), is(24.3f)); + + var = genVars.get("an_pl_q"); + assertNotNull(var); + assertThat(var.getName(), is("q")); + assertThat(var.getUnits(), is("kg kg**-1")); + assertThat(var.getLongName(), is("Specific humidity")); + assertThat(var.getStandardName(), is("specific_humidity")); + assertThat(var.getFillValue(), is(24.3e-5f)); + assertThat(var.is3d(), is(true)); + + var = genVars.get("an_pl_clwc"); + assertNotNull(var); + assertThat(var.getName(), is("clwc")); + assertThat(var.getUnits(), is("kg kg**-1")); + assertThat(var.getLongName(), is("Specific cloud liquid water content")); + assertThat(var.getStandardName(), is(nullValue())); + assertThat(var.getFillValue(), is(9.96921E36F)); + assertThat(var.is3d(), is(true)); + + var = genVars.get("an_pl_crwc"); + assertNotNull(var); + assertThat(var.getName(), is("crwc")); + assertThat(var.getUnits(), is("kg kg**-1")); + assertThat(var.getLongName(), is("Specific rain water content")); + assertThat(var.getStandardName(), is(nullValue())); + assertThat(var.getFillValue(), is(9.96921E36F)); + assertThat(var.is3d(), is(true)); + + var = genVars.get("an_sfc_tclw"); + assertNotNull(var); + assertThat(var.getName(), is("tclw")); + assertThat(var.getUnits(), is("kg m**-2")); + assertThat(var.getLongName(), is("Total column cloud liquid water")); + assertThat(var.getStandardName(), is(nullValue())); + assertThat(var.getFillValue(), is(9.96921E36F)); + assertThat(var.is3d(), is(false)); + + var = genVars.get("an_sfc_tciw"); + assertNotNull(var); + assertThat(var.getName(), is("tciw")); + assertThat(var.getUnits(), is("kg m**-2")); + assertThat(var.getLongName(), is("Total column cloud ice water")); + assertThat(var.getStandardName(), is(nullValue())); + assertThat(var.getFillValue(), is(9.96921E36F)); + assertThat(var.is3d(), is(false)); + + var = genVars.get("an_sfc_tcrw"); + assertNotNull(var); + assertThat(var.getName(), is("tcrw")); + assertThat(var.getUnits(), is("kg m**-2")); + assertThat(var.getLongName(), is("Total column rain water")); + assertThat(var.getStandardName(), is(nullValue())); + assertThat(var.getFillValue(), is(9.96921E36F)); + assertThat(var.is3d(), is(false)); + + var = genVars.get("an_sfc_tcsw"); + assertNotNull(var); + assertThat(var.getName(), is("tcsw")); + assertThat(var.getUnits(), is("kg m**-2")); + assertThat(var.getLongName(), is("Total column snow water")); + assertThat(var.getStandardName(), is(nullValue())); + assertThat(var.getFillValue(), is(9.96921E36F)); + assertThat(var.is3d(), is(false)); + } + + private static Path output(String XML) throws JDOMException, IOException { + final Element rootElement = TestUtil.createDomElement(XML); + final Path configPath = Jimfs.newFileSystem().getPath("config"); + final Path xmlPath = configPath.resolve(Era5PostProcessingPlugin.ERA5_POST_PROCESSING_GENERAL_INFO_XML); + Files.createDirectories(configPath); + try (OutputStream os = Files.newOutputStream(xmlPath)) { + new XMLOutputter(Format.getPrettyFormat()) + .output(rootElement, os); + } + return configPath; + } +} + diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingTest.java index 21d1a43f9..9c92df8a9 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingTest.java @@ -36,12 +36,6 @@ public void testGetInterpolationContext() { interpolator = context.get(2, 1); assertEquals(834, interpolator.getXMin()); assertEquals(247, interpolator.getYMin()); - - final Rectangle era5Region = context.getEra5Region(); - assertEquals(834, era5Region.x); - assertEquals(247, era5Region.y); - assertEquals(3, era5Region.width); - assertEquals(3, era5Region.height); } @Test @@ -58,12 +52,6 @@ public void testGetInterpolationContext_singlePixel() { BilinearInterpolator interpolator = context.get(0, 0); assertEquals(835, interpolator.getXMin()); assertEquals(247, interpolator.getYMin()); - - final Rectangle era5Region = context.getEra5Region(); - assertEquals(835, era5Region.x); - assertEquals(247, era5Region.y); - assertEquals(2, era5Region.width); - assertEquals(2, era5Region.height); } @Test @@ -120,6 +108,7 @@ public void testGetLonMin() { assertEquals(1344, Era5PostProcessing.getEra5LonMin(-23.8f)); assertEquals(1438, Era5PostProcessing.getEra5LonMin(-0.26f)); assertEquals(1439, Era5PostProcessing.getEra5LonMin(-0.18f)); + assertEquals(1439, Era5PostProcessing.getEra5LonMin(-0.01236f)); assertEquals(0, Era5PostProcessing.getEra5LonMin(0.f)); assertEquals(173, Era5PostProcessing.getEra5LonMin(43.32f)); assertEquals(718, Era5PostProcessing.getEra5LonMin(179.58f)); @@ -171,6 +160,10 @@ public void testCreateInterpolator() { assertEquals(0.2799999713897705, interpolator.getA(), 1e-8); assertEquals(1.0, interpolator.getB(), 1e-8); + interpolator = Era5PostProcessing.createInterpolator(-0.01236f, 0.f, 1439, 359); + assertEquals(1439, interpolator.getXMin(), 1e-8); + assertEquals(359, interpolator.getYMin(), 1e-8); + interpolator = Era5PostProcessing.createInterpolator(43.32f, -22.19f, 173, 448); assertEquals(0.279998779296875, interpolator.getA(), 1e-8); assertEquals(0.7600021362304688, interpolator.getB(), 1e-8); diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/InterpolationContextTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/InterpolationContextTest.java index a990f98f9..c5106a4cd 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/InterpolationContextTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/InterpolationContextTest.java @@ -1,9 +1,8 @@ package com.bc.fiduceo.post.plugin.era5; +import com.bc.fiduceo.core.IntRange; import org.junit.Test; -import java.awt.*; - import static org.junit.Assert.*; public class InterpolationContextTest { @@ -85,15 +84,109 @@ public void testGet_outOfBounds() { } @Test - public void testSetGetEra5Region() { - final Rectangle rectangle = new Rectangle(5, 6, 7, 8); - final InterpolationContext context = new InterpolationContext(6, 4); - - context.setEra5Region(rectangle); - final Rectangle era5Region = context.getEra5Region(); - assertEquals(5, era5Region.x); - assertEquals(6, era5Region.y); - assertEquals(7, era5Region.width); - assertEquals(8, era5Region.height); + public void testGetXRanges_emptyContext() { + final InterpolationContext context = new InterpolationContext(5, 3); + + final IntRange[] xRanges = context.getXRanges(); + assertEquals(2, xRanges.length); + assertEquals(Integer.MAX_VALUE, xRanges[0].getMin()); + assertEquals(Integer.MIN_VALUE, xRanges[0].getMax()); + assertNull(xRanges[1]); + } + + @Test + public void testGetXRanges_oneInterpolator() { + final InterpolationContext context = new InterpolationContext(4, 5); + + final BilinearInterpolator interpolator = new BilinearInterpolator(0.3, 0.5, 6, 7); + context.set(0, 0, interpolator); + + final IntRange[] xRanges = context.getXRanges(); + assertEquals(2, xRanges.length); + assertEquals(6, xRanges[0].getMin()); + assertEquals(7, xRanges[0].getMax()); + + assertEquals(0, interpolator.getRelXMin()); + assertEquals(0, interpolator.getRelYMin()); + } + + @Test + public void testGetXRanges_threeInterpolators() { + final InterpolationContext context = new InterpolationContext(4, 5); + + BilinearInterpolator interpolator = new BilinearInterpolator(0.3, 0.5, 6, 7); + context.set(0, 0, interpolator); + + interpolator = new BilinearInterpolator(0.3, 0.5, 7, 8); + context.set(1, 1, interpolator); + + interpolator = new BilinearInterpolator(0.3, 0.5, 8, 9); + context.set(2, 2, interpolator); + + final IntRange[] xRanges = context.getXRanges(); + assertEquals(2, xRanges.length); + assertEquals(6, xRanges[0].getMin()); + assertEquals(9, xRanges[0].getMax()); + } + + @Test + public void testGetXRanges_threeInterpolators_antimeridianCase() { + final InterpolationContext context = new InterpolationContext(4, 5); + + BilinearInterpolator interpolator = new BilinearInterpolator(0.3, 0.5, 1438, 7); + context.set(0, 0, interpolator); + + interpolator = new BilinearInterpolator(0.3, 0.5, 1439, 8); + context.set(1, 1, interpolator); + + interpolator = new BilinearInterpolator(0.3, 0.5, 0, 9); + context.set(2, 2, interpolator); + + final IntRange[] xRanges = context.getXRanges(); + assertEquals(2, xRanges.length); + assertEquals(1438, xRanges[0].getMin()); + assertEquals(1439, xRanges[0].getMax()); + + assertEquals(0, xRanges[1].getMin()); + assertEquals(1, xRanges[1].getMax()); + } + + @Test + public void testGetYRange_emptyContext() { + final InterpolationContext context = new InterpolationContext(5, 3); + + final IntRange yRange = context.getYRange(); + assertEquals(Integer.MAX_VALUE, yRange.getMin()); + assertEquals(Integer.MIN_VALUE, yRange.getMax()); + } + + @Test + public void testGetYRange_oneInterpolator() { + final InterpolationContext context = new InterpolationContext(4, 5); + + final BilinearInterpolator interpolator = new BilinearInterpolator(0.3, 0.5, 6, 7); + context.set(0, 0, interpolator); + + final IntRange yRange = context.getYRange(); + assertEquals(7, yRange.getMin()); + assertEquals(8, yRange.getMax()); + } + + @Test + public void testGetYRange_threeInterpolators() { + final InterpolationContext context = new InterpolationContext(4, 5); + + BilinearInterpolator interpolator = new BilinearInterpolator(0.3, 0.5, 6, 7); + context.set(0, 0, interpolator); + + interpolator = new BilinearInterpolator(0.3, 0.5, 7, 7); + context.set(1, 0, interpolator); + + interpolator = new BilinearInterpolator(0.3, 0.5, 7, 8); + context.set(1, 1, interpolator); + + final IntRange yRange = context.getYRange(); + assertEquals(7, yRange.getMin()); + assertEquals(9, yRange.getMax()); } } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsConfigurationTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsConfigurationTest.java index b23cc3281..a4c26208b 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsConfigurationTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsConfigurationTest.java @@ -16,19 +16,19 @@ public void setUp() { @Test public void testConstructionAndDefaultValues() { - assertEquals("nwp_q", config.get_an_q_name()); - assertEquals("nwp_t", config.get_an_t_name()); - assertEquals("nwp_o3", config.get_an_o3_name()); - assertEquals("nwp_lnsp", config.get_an_lnsp_name()); - assertEquals("nwp_t2m", config.get_an_t2m_name()); - assertEquals("nwp_siconc", config.get_an_siconc_name()); - assertEquals("nwp_u10", config.get_an_u10_name()); - assertEquals("nwp_v10", config.get_an_v10_name()); - assertEquals("nwp_msl", config.get_an_msl_name()); - assertEquals("nwp_skt", config.get_an_skt_name()); - assertEquals("nwp_sst", config.get_an_sst_name()); - assertEquals("nwp_tcc", config.get_an_tcc_name()); - assertEquals("nwp_tcwv", config.get_an_tcwv_name()); + assertEquals("nwp_q", config.getVarName("an_ml_q")); + assertEquals("nwp_t", config.getVarName("an_ml_t")); + assertEquals("nwp_o3", config.getVarName("an_ml_o3")); + assertEquals("nwp_lnsp", config.getVarName("an_ml_lnsp")); + assertEquals("nwp_t2m", config.getVarName("an_sfc_t2m") ); + assertEquals("nwp_siconc", config.getVarName("an_sfc_siconc") ); + assertEquals("nwp_u10", config.getVarName("an_sfc_u10") ); + assertEquals("nwp_v10", config.getVarName("an_sfc_v10") ); + assertEquals("nwp_msl", config.getVarName("an_sfc_msl")); + assertEquals("nwp_skt", config.getVarName("an_sfc_skt")); + assertEquals("nwp_sst", config.getVarName("an_sfc_sst")); + assertEquals("nwp_tcc", config.getVarName("an_sfc_tcc")); + assertEquals("nwp_tcwv", config.getVarName("an_sfc_tcwv")); assertEquals(-1, config.get_x_dim()); assertEquals(-1, config.get_y_dim()); @@ -44,81 +44,33 @@ public void testConstructionAndDefaultValues() { } @Test - public void testSetGet_an_q() { - config.set_an_q_name("anku"); - assertEquals("anku", config.get_an_q_name()); - } - - @Test - public void testSetGet_an_t() { - config.set_an_t_name("tee"); - assertEquals("tee", config.get_an_t_name()); - } - - @Test - public void testSetGet_an_o3() { - config.set_an_o3_name("ozzi"); - assertEquals("ozzi", config.get_an_o3_name()); - } - - @Test - public void testSetGet_an_lnsp() { - config.set_an_lnsp_name("pratt"); - assertEquals("pratt", config.get_an_lnsp_name()); - } - - @Test - public void testSetGet_an_t2m() { - config.set_an_t2m_name("tempi"); - assertEquals("tempi", config.get_an_t2m_name()); - } - - @Test - public void testSetGet_an_u10() { - config.set_an_u10_name("windu"); - assertEquals("windu", config.get_an_u10_name()); - } - - @Test - public void testSetGet_an_v10() { - config.set_an_v10_name("Vicky"); - assertEquals("Vicky", config.get_an_v10_name()); - } - - @Test - public void testSetGet_an_siconc() { - config.set_an_siconc_name("sieglinde"); - assertEquals("sieglinde", config.get_an_siconc_name()); - } - - @Test - public void testSetGet_an_mslc() { - config.set_an_msl_name("meanSurf"); - assertEquals("meanSurf", config.get_an_msl_name()); - } - - @Test - public void testSetGet_an_skt() { - config.set_an_skt_name("scinny"); - assertEquals("scinny", config.get_an_skt_name()); - } - - @Test - public void testSetGet_an_sst() { - config.set_an_sst_name("seaTemp"); - assertEquals("seaTemp", config.get_an_sst_name()); - } - - @Test - public void testSetGet_an_tcc() { - config.set_an_tcc_name("cloudCover"); - assertEquals("cloudCover", config.get_an_tcc_name()); - } - - @Test - public void testSetGet_an_tcwv() { - config.set_an_tcwv_name("steamy"); - assertEquals("steamy", config.get_an_tcwv_name()); + public void testSetGetVarName() { + config.setVarName("an_ml_q","anku"); + assertEquals("anku", config.getVarName("an_ml_q")); + config.setVarName("an_ml_t","tee"); + assertEquals("tee", config.getVarName("an_ml_t")); + config.setVarName("an_ml_o3", "ozzi"); + assertEquals("ozzi", config.getVarName("an_ml_o3")); + config.setVarName("an_ml_lnsp", "pratt"); + assertEquals("pratt", config.getVarName("an_ml_lnsp")); + config.setVarName("an_sfc_t2m", "tempi"); + assertEquals("tempi", config.getVarName("an_sfc_t2m") ); + config.setVarName("an_sfc_u10", "windu"); + assertEquals("windu", config.getVarName("an_sfc_u10") ); + config.setVarName("an_sfc_v10", "Vicky"); + assertEquals("Vicky", config.getVarName("an_sfc_v10") ); + config.setVarName("an_sfc_siconc", "sieglinde"); + assertEquals("sieglinde", config.getVarName("an_sfc_siconc") ); + config.setVarName("an_sfc_msl", "meanSurf"); + assertEquals("meanSurf", config.getVarName("an_sfc_msl")); + config.setVarName("an_sfc_skt", "scinny"); + assertEquals("scinny", config.getVarName("an_sfc_skt")); + config.setVarName("an_sfc_sst", "seaTemp"); + assertEquals("seaTemp", config.getVarName("an_sfc_sst")); + config.setVarName("an_sfc_tcc", "cloudCover"); + assertEquals("cloudCover", config.getVarName("an_sfc_tcc")); + config.setVarName("an_sfc_tcwv", "steamy"); + assertEquals("steamy", config.getVarName("an_sfc_tcwv")); } @Test @@ -179,44 +131,44 @@ public void testGetVariablesWithReplacement() { config.set_time_variable_name("{sensor-ref}_acquisition-time"); assertEquals("avhrr-n14_acquisition-time", config.get_time_variable_name()); - config.set_an_q_name("a_{sensor-ref}_n_q"); - assertEquals("a_avhrr-n14_n_q", config.get_an_q_name()); + config.setVarName("an_ml_q", "a_{sensor-ref}_n_q"); + assertEquals("a_avhrr-n14_n_q", config.getVarName("an_ml_q")); - config.set_an_t_name("an_{sensor-ref}_t"); - assertEquals("an_avhrr-n14_t", config.get_an_t_name()); + config.setVarName("an_ml_t","an_{sensor-ref}_t"); + assertEquals("an_avhrr-n14_t", config.getVarName("an_ml_t")); - config.set_an_o3_name("{sensor-ref}_o3"); - assertEquals("avhrr-n14_o3", config.get_an_o3_name()); + config.setVarName("an_ml_o3", "{sensor-ref}_o3"); + assertEquals("avhrr-n14_o3", config.getVarName("an_ml_o3")); - config.set_an_lnsp_name("{sensor-ref}_lnsp"); - assertEquals("avhrr-n14_lnsp", config.get_an_lnsp_name()); + config.setVarName("an_ml_lnsp", "{sensor-ref}_lnsp"); + assertEquals("avhrr-n14_lnsp", config.getVarName("an_ml_lnsp")); - config.set_an_t2m_name("{sensor-ref}_t2m"); - assertEquals("avhrr-n14_t2m", config.get_an_t2m_name()); + config.setVarName("an_sfc_t2m", "{sensor-ref}_t2m"); + assertEquals("avhrr-n14_t2m", config.getVarName("an_sfc_t2m") ); - config.set_an_siconc_name("{sensor-ref}_siconc"); - assertEquals("avhrr-n14_siconc", config.get_an_siconc_name()); + config.setVarName("an_sfc_siconc", "{sensor-ref}_siconc"); + assertEquals("avhrr-n14_siconc", config.getVarName("an_sfc_siconc") ); - config.set_an_u10_name("{sensor-ref}_u10"); - assertEquals("avhrr-n14_u10", config.get_an_u10_name()); + config.setVarName("an_sfc_u10", "{sensor-ref}_u10"); + assertEquals("avhrr-n14_u10", config.getVarName("an_sfc_u10") ); - config.set_an_v10_name("{sensor-ref}_v10"); - assertEquals("avhrr-n14_v10", config.get_an_v10_name()); + config.setVarName("an_sfc_v10", "{sensor-ref}_v10"); + assertEquals("avhrr-n14_v10", config.getVarName("an_sfc_v10") ); - config.set_an_msl_name("{sensor-ref}_msl"); - assertEquals("avhrr-n14_msl", config.get_an_msl_name()); + config.setVarName("an_sfc_msl", "{sensor-ref}_msl"); + assertEquals("avhrr-n14_msl", config.getVarName("an_sfc_msl")); - config.set_an_skt_name("{sensor-ref}_skt"); - assertEquals("avhrr-n14_skt", config.get_an_skt_name()); + config.setVarName("an_sfc_skt", "{sensor-ref}_skt"); + assertEquals("avhrr-n14_skt", config.getVarName("an_sfc_skt")); - config.set_an_sst_name("{sensor-ref}_sst"); - assertEquals("avhrr-n14_sst", config.get_an_sst_name()); + config.setVarName("an_sfc_sst", "{sensor-ref}_sst"); + assertEquals("avhrr-n14_sst", config.getVarName("an_sfc_sst")); - config.set_an_tcc_name("{sensor-ref}_tcc"); - assertEquals("avhrr-n14_tcc", config.get_an_tcc_name()); + config.setVarName("an_sfc_tcc", "{sensor-ref}_tcc"); + assertEquals("avhrr-n14_tcc", config.getVarName("an_sfc_tcc")); - config.set_an_tcwv_name("{sensor-ref}_tcwv"); - assertEquals("avhrr-n14_tcwv", config.get_an_tcwv_name()); + config.setVarName("an_sfc_tcwv", "{sensor-ref}_tcwv"); + assertEquals("avhrr-n14_tcwv", config.getVarName("an_sfc_tcwv")); } @Test @@ -274,7 +226,6 @@ public void testVerify_y_dim() { } catch (IllegalArgumentException expected) { } } - @Test public void testVerify_y_dim_name() { prepareConfig(); @@ -287,6 +238,18 @@ public void testVerify_y_dim_name() { } } + @Test + public void testVerify_z_dim() { + prepareConfig(); + config.set_z_dim(-2); + + try { + config.verify(); + fail("IllegalArgumentException expected"); + } catch (IllegalArgumentException expected) { + } + } + @Test public void testVerify_z_dim_name() { prepareConfig(); @@ -299,6 +262,17 @@ public void testVerify_z_dim_name() { } } + @Test + public void test_dont_verify_z_dim_if_z_dim_name_is_not_set() { + //preparation + prepareConfig(); + config.set_z_dim(-2); + config.set_z_dim_name(null); + + //execution + config.verify(); + } + @Test public void testVerify_nwp_time_variable_name() { prepareConfig(); diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsTest.java index b894a7718..3229701ae 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsTest.java @@ -1,7 +1,6 @@ package com.bc.fiduceo.post.plugin.era5; import com.bc.fiduceo.FiduceoConstants; -import org.checkerframework.checker.units.qual.A; import org.junit.Test; import ucar.ma2.Array; import ucar.ma2.DataType; @@ -9,13 +8,18 @@ import ucar.nc2.Dimension; import ucar.nc2.NetcdfFile; import ucar.nc2.NetcdfFileWriter; -import ucar.nc2.Variable; import java.awt.*; +import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.*; @@ -26,7 +30,7 @@ public class SatelliteFieldsTest { public void testGetVariables() { final SatelliteFields satelliteFields = new SatelliteFields(); final SatelliteFieldsConfiguration config = new SatelliteFieldsConfiguration(); - config.set_an_skt_name("Skate"); + config.setVarName("an_sfc_skt", "Skate"); final Map variables = satelliteFields.getVariables(config); assertEquals(13, variables.size()); @@ -46,6 +50,48 @@ public void testGetVariables() { assertFalse(template.is3d()); } + @Test + public void testGetVariables_withGeneralizedVariables() { + //preparation + final SatelliteFieldsConfiguration config = new SatelliteFieldsConfiguration(); + final TemplateVariable tempVar1 = new TemplateVariable("skt", "u1", "ln1", "sn1", true); + final TemplateVariable tempVar2 = new TemplateVariable("kbb", "u2", "ln2", null, false); + tempVar1.setFill_value(123.4f); + tempVar2.setFill_value(123.4e-1f); + final HashMap generalizedVars = new HashMap<>(); + generalizedVars.put("an_sfc_skt", tempVar1); + generalizedVars.put("an_sfc_kbb", tempVar2); + config.setGeneralizedVariables(generalizedVars); + + config.setSensorRef("DerSensor"); + config.setVarName("an_sfc_skt", "{sensor-ref}.und-skt"); + config.setVarName("an_sfc_kbb", "{sensor-ref}.morgenrot"); + + final SatelliteFields satelliteFields = new SatelliteFields(); + + //execution + final Map variables = satelliteFields.getVariables(config); + + //verification + assertEquals(2, variables.size()); + + TemplateVariable template = variables.get("an_sfc_skt"); + assertEquals("sn1", template.getStandardName()); + assertEquals("u1", template.getUnits()); + assertEquals("ln1", template.getLongName()); + assertEquals("DerSensor.und-skt", template.getName()); + assertThat(template.getFillValue(), is(123.4f)); + assertTrue(template.is3d()); + + template = variables.get("an_sfc_kbb"); + assertNull(template.getStandardName()); + assertEquals("u2", template.getUnits()); + assertEquals("ln2", template.getLongName()); + assertEquals("DerSensor.morgenrot", template.getName()); + assertThat(template.getFillValue(), is(12.34f)); + assertFalse(template.is3d()); + } + @Test public void testSetGetDimensions_2D() { final NetcdfFile ncFile = mock(NetcdfFile.class); diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields_IO_Test.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields_IO_Test.java deleted file mode 100644 index 9f430dab4..000000000 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields_IO_Test.java +++ /dev/null @@ -1,150 +0,0 @@ -package com.bc.fiduceo.post.plugin.era5; - - -import com.bc.fiduceo.IOTestRunner; -import com.bc.fiduceo.TestUtil; -import org.checkerframework.checker.units.qual.C; -import org.junit.Test; -import org.junit.runner.RunWith; -import ucar.ma2.Array; -import ucar.ma2.InvalidRangeException; -import ucar.nc2.NetcdfFile; -import ucar.nc2.Variable; - -import java.awt.*; -import java.io.File; -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -@RunWith(IOTestRunner.class) -public class SatelliteFields_IO_Test { - - @Test - public void testReadSubset_2D() throws IOException, InvalidRangeException { - final File anMlFile = TestUtil.getTestDataFileAsserted("era-5/v1/an_ml/2008/05/30/ecmwf-era5_oper_an_ml_200805301100.lnsp.nc"); - - try (NetcdfFile netcdfFile = NetcdfFile.open(anMlFile.getAbsolutePath())) { - final Variable lnsp = netcdfFile.findVariable("lnsp"); - assertNotNull(lnsp); - Array lnspFull = lnsp.read().reduce(); - - final Rectangle era5Positions = new Rectangle(1200, 400, 3, 3); - - final Array lnspArray = SatelliteFields.readSubset(1, era5Positions, new VariableCache.CacheEntry(lnsp, lnspFull)); - assertNotNull(lnspArray); - - final int[] shape = lnspArray.getShape(); - assertEquals(2, shape.length); - assertEquals(3, shape[0]); - assertEquals(3, shape[1]); - - assertEquals(11.503242492675781f, lnspArray.getFloat(0), 1e-8); - assertEquals(11.503653526306152f, lnspArray.getFloat(1), 1e-8); - assertEquals(11.505866050720215f, lnspArray.getFloat(2), 1e-8); - } - } - - @Test - public void testReadSubset_3D() throws IOException, InvalidRangeException { - final File o3File = TestUtil.getTestDataFileAsserted("era-5/v1/an_ml/2008/05/30/ecmwf-era5_oper_an_ml_200805301100.o3.nc"); - - try (NetcdfFile netcdfFile = NetcdfFile.open(o3File.getAbsolutePath())) { - final Variable o3 = netcdfFile.findVariable("o3"); - assertNotNull(o3); - Array o3Full = o3.read().reduce(); - - final Rectangle era5Positions = new Rectangle(1201, 401, 3, 3); - - final Array lnspArray = SatelliteFields.readSubset(137, era5Positions, new VariableCache.CacheEntry(o3, o3Full)); - assertNotNull(lnspArray); - - final int[] shape = lnspArray.getShape(); - assertEquals(3, shape.length); - assertEquals(137, shape[0]); - assertEquals(3, shape[1]); - assertEquals(3, shape[2]); - - assertEquals(1.9680332741245365E-7f, lnspArray.getFloat(1), 1e-8); - assertEquals(1.9680332741245365E-7f, lnspArray.getFloat(2), 1e-8); - assertEquals(1.9680332741245365E-7f, lnspArray.getFloat(3), 1e-8); - } - } - - @Test - public void testReadSubset_2D_antiMeridianOverlap_right() throws IOException, InvalidRangeException { - final File anMlFile = TestUtil.getTestDataFileAsserted("era-5/v1/an_ml/2008/06/02/ecmwf-era5_oper_an_ml_200806021000.lnsp.nc"); - - try (NetcdfFile netcdfFile = NetcdfFile.open(anMlFile.getAbsolutePath())) { - final Variable lnsp = netcdfFile.findVariable("lnsp"); - assertNotNull(lnsp); - Array lnspFull = lnsp.read().reduce(); - - final Rectangle era5Positions = new Rectangle(1439, 402, 3, 3); - - final Array lnspArray = SatelliteFields.readSubset(1, era5Positions, new VariableCache.CacheEntry(lnsp, lnspFull)); - assertNotNull(lnspArray); - - final int[] shape = lnspArray.getShape(); - assertEquals(2, shape.length); - assertEquals(3, shape[0]); - assertEquals(3, shape[1]); - - assertEquals(11.529290199279785f, lnspArray.getFloat(0), 1e-8); - assertEquals(11.529301643371582f, lnspArray.getFloat(1), 1e-8); - assertEquals(11.529279708862305f, lnspArray.getFloat(2), 1e-8); - } - } - - @Test - public void testReadSubset_2D_antiMeridianOverlap_right_JasminCrash() throws IOException, InvalidRangeException { - final File anMlFile = TestUtil.getTestDataFileAsserted("era-5/v1/an_ml/2008/06/02/ecmwf-era5_oper_an_ml_200806021000.lnsp.nc"); - - try (NetcdfFile netcdfFile = NetcdfFile.open(anMlFile.getAbsolutePath())) { - final Variable lnsp = netcdfFile.findVariable("lnsp"); - assertNotNull(lnsp); - Array lnspFull = lnsp.read().reduce(); - - final Rectangle era5Positions = new Rectangle(1438, 66, 2, 2); - - final Array lnspArray = SatelliteFields.readSubset(1, era5Positions, new VariableCache.CacheEntry(lnsp, lnspFull)); - assertNotNull(lnspArray); - - final int[] shape = lnspArray.getShape(); - assertEquals(2, shape.length); - assertEquals(2, shape[0]); - assertEquals(2, shape[1]); - - assertEquals(11.532238960266113f, lnspArray.getFloat(0), 1e-8); - assertEquals(11.532217025756836f, lnspArray.getFloat(1), 1e-8); - assertEquals(11.532645225524902f, lnspArray.getFloat(2), 1e-8); - assertEquals(11.532743453979492f, lnspArray.getFloat(3), 1e-8); - } - } - - @Test - public void testReadSubset_2D_antiMeridianOverlap_left() throws IOException, InvalidRangeException { - final File anMlFile = TestUtil.getTestDataFileAsserted("era-5/v1/an_sfc/2000/05/28/ecmwf-era5_oper_an_sfc_200005281700.10u.nc"); - - try (NetcdfFile netcdfFile = NetcdfFile.open(anMlFile.getAbsolutePath())) { - final Variable lnsp = netcdfFile.findVariable("u10"); - assertNotNull(lnsp); - Array lnspFull = lnsp.read().reduce(); - - final Rectangle era5Positions = new Rectangle(-1, 403, 3, 3); - - final Array lnspArray = SatelliteFields.readSubset(1, era5Positions, new VariableCache.CacheEntry(lnsp, lnspFull)); - assertNotNull(lnspArray); - - final int[] shape = lnspArray.getShape(); - assertEquals(2, shape.length); - assertEquals(3, shape[0]); - assertEquals(3, shape[1]); - - assertEquals(-6.088078022003174f, lnspArray.getFloat(0), 1e-8); - assertEquals(-5.904586315155029f, lnspArray.getFloat(1), 1e-8); - assertEquals(-6.2205610275268555f, lnspArray.getFloat(2), 1e-8); - } - } -} diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/VariableCacheTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/VariableCacheTest.java index 199b93b77..29a47ab34 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/VariableCacheTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/VariableCacheTest.java @@ -28,7 +28,9 @@ public void setUp() throws IOException { final File era5RootDir = new File(testDataDirectory, "era-5" + File.separator + "v1"); assertTrue(era5RootDir.isDirectory()); - final Era5Archive era5Archive = new Era5Archive(era5RootDir.getAbsolutePath(), Era5Collection.ERA_5); + final Configuration config = new Configuration(); + config.setNWPAuxDir(era5RootDir.getAbsolutePath()); + final Era5Archive era5Archive = new Era5Archive(config, Era5Collection.ERA_5); variableCache = new VariableCache(era5Archive, 3); } @@ -40,58 +42,67 @@ public void tearDown() throws IOException { @Test public void testGet() throws IOException { - VariableCache.CacheEntry cacheEntry = variableCache.get("an_ml_lnsp", 1212145200); - assertEquals("time latitude longitude", cacheEntry.variable.getDimensionsString()); - assertEquals("Logarithm of surface pressure", NetCDFUtils.getAttributeString(cacheEntry.variable, "long_name", null)); + Variable variable; - int[] shape = cacheEntry.array.getShape(); - assertEquals(2, shape.length); - assertEquals(DataType.SHORT, cacheEntry.array.getDataType()); + variable = variableCache.get("an_ml_lnsp", 1212145200); + assertEquals("time latitude longitude", variable.getDimensionsString()); + assertEquals("Logarithm of surface pressure", NetCDFUtils.getAttributeString(variable, "long_name", null)); - cacheEntry = variableCache.get("an_ml_q", 1212145200); - assertEquals("time level latitude longitude", cacheEntry.variable.getDimensionsString()); - assertEquals(3.786489628510026E-7, NetCDFUtils.getAttributeFloat(cacheEntry.variable, "scale_factor", Float.NaN), 1e-8); - - shape = cacheEntry.array.getShape(); + int[] shape = variable.getShape(); + assertArrayEquals(new int[]{1, 721, 1440}, shape); assertEquals(3, shape.length); - assertEquals(DataType.SHORT, cacheEntry.array.getDataType()); + assertEquals(DataType.SHORT, variable.getDataType()); + + variable = variableCache.get("an_ml_q", 1212145200); + assertEquals("time level latitude longitude", variable.getDimensionsString()); + assertEquals(3.786489628510026E-7, NetCDFUtils.getAttributeFloat(variable, "scale_factor", Float.NaN), 1e-8); + + shape = variable.getShape(); + assertArrayEquals(new int[]{1, 137, 721, 1440}, shape); + assertEquals(4, shape.length); + assertEquals(DataType.SHORT, variable.getDataType()); } @Test public void testCallGetTwice() throws IOException { - final VariableCache.CacheEntry cacheEntry_1 = variableCache.get("an_ml_o3", 1212400800); + final Variable variable_1 = variableCache.get("an_ml_o3", 1212400800); - final VariableCache.CacheEntry cacheEntry_2 = variableCache.get("an_ml_o3", 1212400800); - assertSame(cacheEntry_1, cacheEntry_2); + final Variable variable_2 = variableCache.get("an_ml_o3", 1212400800); + + assertSame(variable_1, variable_2); } @Test public void testCallGetTwice_closeInbetween() throws IOException { - VariableCache.CacheEntry cacheEntry_1 = variableCache.get("an_sfc_t2m", 1212145200); + final Variable variable_1 = variableCache.get("an_sfc_t2m", 1212145200); variableCache.close(); - VariableCache.CacheEntry cacheEntry_2 = variableCache.get("an_sfc_t2m", 1212145200); - assertNotSame(cacheEntry_1, cacheEntry_2); + final Variable variable_2 = variableCache.get("an_sfc_t2m", 1212145200); + + assertNotSame(variable_1, variable_2); } @Test public void testGet_removeFunctionalityOnFullCache() throws IOException, InterruptedException { - VariableCache.CacheEntry cached_u10 = variableCache.get("an_sfc_u10", 1212400800); - sleep(50); + final int secondsOfOneHour = 60 * 60; + final int era5TimeStamp_start = 959533200; // timestamp for 17:00 2000-05-28 + int era5TimeStamp = era5TimeStamp_start; + + Variable cached_u10 = variableCache.get("an_sfc_u10", era5TimeStamp); - variableCache.get("an_sfc_v10", 1212400800); - sleep(50); + era5TimeStamp += secondsOfOneHour; + variableCache.get("an_sfc_u10", era5TimeStamp); // one hour later - variableCache.get("an_sfc_siconc", 1212400800); - sleep(50); + era5TimeStamp += secondsOfOneHour; + variableCache.get("an_sfc_u10", era5TimeStamp); // one hour later // now the cache is full tb 2020-11-25 - variableCache.get("an_sfc_msl", 1212400800); - sleep(50); + era5TimeStamp += secondsOfOneHour; + variableCache.get("an_sfc_u10", era5TimeStamp); // one hour later // now the first u10 variable is removed from cache and opening it again must result in a new object tb 2020-11-25 - VariableCache.CacheEntry cached_u10_new = variableCache.get("an_sfc_u10", 1212400800); + Variable cached_u10_new = variableCache.get("an_sfc_u10", era5TimeStamp_start); assertNotSame(cached_u10, cached_u10_new); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/VariableUtilsTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/VariableUtilsTest.java index cecd55352..8333fbac7 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/VariableUtilsTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/VariableUtilsTest.java @@ -22,6 +22,7 @@ public void testAddAttributes() { verify(variable, times(1)).addAttribute(new Attribute("long_name", "a_long_name")); verify(variable, times(1)).addAttribute(new Attribute("standard_name", "a_standard_name")); verify(variable, times(1)).addAttribute(new Attribute("_FillValue", 9.96921E36f)); + verify(variable, times(1)).addAttribute(new Attribute("missing_value", 9.96921E36f)); verifyNoMoreInteractions(variable); } @@ -35,6 +36,7 @@ public void testAddAttributes_missingStandarName() { verify(variable, times(1)).addAttribute(new Attribute("units", "gramm")); verify(variable, times(1)).addAttribute(new Attribute("long_name", "Heffalump")); verify(variable, times(1)).addAttribute(new Attribute("_FillValue", 9.96921E36f)); + verify(variable, times(1)).addAttribute(new Attribute("missing_value", 9.96921E36f)); verifyNoMoreInteractions(variable); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourcePluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourcePluginTest.java index 3e1663390..b02ea3e4b 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourcePluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourcePluginTest.java @@ -1,8 +1,8 @@ package com.bc.fiduceo.post.plugin.gruan_uleic; import com.bc.fiduceo.post.PostProcessing; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; @@ -29,7 +29,7 @@ public void testGetPostProcessingName() { public void testCreatePostProcessing() throws JDOMException, IOException { final Element rootElement = AddGruanSourceTest.createFullConfigElement(); - final PostProcessing postProcessing = plugin.createPostProcessing(rootElement); + final PostProcessing postProcessing = plugin.createPostProcessing(rootElement, null); assertTrue(postProcessing instanceof AddGruanSource); } } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourceTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourceTest.java index da3f7b004..b4e166f65 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourceTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/gruan_uleic/AddGruanSourceTest.java @@ -7,8 +7,8 @@ import com.bc.fiduceo.post.PostProcessingConfig; import com.bc.fiduceo.post.PostProcessingContext; import org.checkerframework.checker.units.qual.A; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; import ucar.ma2.DataType; diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/hirs/flag/HirsL1CloudyFlagsPluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/hirs/flag/HirsL1CloudyFlagsPluginTest.java index f7b76c818..d94e96dc1 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/hirs/flag/HirsL1CloudyFlagsPluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/hirs/flag/HirsL1CloudyFlagsPluginTest.java @@ -37,7 +37,7 @@ import com.bc.fiduceo.post.PostProcessing; import com.google.common.jimfs.Configuration; import com.google.common.jimfs.Jimfs; -import org.jdom.Element; +import org.jdom2.Element; import org.junit.*; import java.nio.file.FileSystem; @@ -78,7 +78,7 @@ public void setUp() throws Exception { @Test public void testCreatePostProcessing() { plugin.setFileSystem(virtualFS); - final PostProcessing postProcessing = plugin.createPostProcessing(element); + final PostProcessing postProcessing = plugin.createPostProcessing(element, null); assertNotNull(postProcessing); assertEquals("com.bc.fiduceo.post.plugin.hirs.flag.HirsL1CloudyFlags", postProcessing.getClass().getTypeName()); @@ -104,7 +104,7 @@ public void testCreatePostProcessing_wrongElementName() throws Exception { element.setName("wrong_name"); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Illegal XML Element. Tagname 'hirs-l1-cloudy-flags' expected.", expected.getMessage()); @@ -116,7 +116,7 @@ public void testCreatePostProcessing_11_1_um_VarNameTextIsMissing() throws Excep element.getChild(TAG_VAR_NAME_BT_11_1_uM).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_BT_11_1_uM + "' expected", expected.getMessage()); @@ -128,7 +128,7 @@ public void testCreatePostProcessing_11_1_um_VarNameTextIsEmpty() throws Excepti element.getChild(TAG_VAR_NAME_BT_11_1_uM).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_BT_11_1_uM + "' expected", expected.getMessage()); @@ -140,7 +140,7 @@ public void testCreatePostProcessing_11_1_um_VarNameElementIsMissing() throws Ex element.removeChild(TAG_VAR_NAME_BT_11_1_uM); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element '" + TAG_VAR_NAME_BT_11_1_uM + "' expected", expected.getMessage()); @@ -152,7 +152,7 @@ public void testCreatePostProcessing_6_5_um_VarNameTextIsMissing() throws Except element.getChild(TAG_VAR_NAME_BT_6_5_uM).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_BT_6_5_uM + "' expected", expected.getMessage()); @@ -164,7 +164,7 @@ public void testCreatePostProcessing_6_5_um_VarNameTextIsEmpty() throws Exceptio element.getChild(TAG_VAR_NAME_BT_6_5_uM).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_BT_6_5_uM + "' expected", expected.getMessage()); @@ -176,7 +176,7 @@ public void testCreatePostProcessing_6_5_um_VarNameElementIsMissing() throws Exc element.removeChild(TAG_VAR_NAME_BT_6_5_uM); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element '" + TAG_VAR_NAME_BT_6_5_uM + "' expected", expected.getMessage()); @@ -188,7 +188,7 @@ public void testCreatePostProcessing_FlagVarNameTextIsMissing() throws Exception element.getChild(TAG_VAR_NAME_CLOUD_FLAGS).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_CLOUD_FLAGS + "' expected", expected.getMessage()); @@ -200,7 +200,7 @@ public void testCreatePostProcessing_FlagVarNameTextIsEmpty() throws Exception { element.getChild(TAG_VAR_NAME_CLOUD_FLAGS).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_CLOUD_FLAGS + "' expected", expected.getMessage()); @@ -212,7 +212,7 @@ public void testCreatePostProcessing_FlagVarNameElementIsMissing() throws Except element.removeChild(TAG_VAR_NAME_CLOUD_FLAGS); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element '" + TAG_VAR_NAME_CLOUD_FLAGS + "' expected", expected.getMessage()); @@ -224,7 +224,7 @@ public void testCreatePostProcessing_DistanceFilePathIsMissing() throws Exceptio element.getChild(TAG_DISTANCE_PRODUCT_FILE_PATH).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element 'point_distance-product-file-path' expected", expected.getMessage()); @@ -236,7 +236,7 @@ public void testCreatePostProcessing_DistanceFilePathTextIsEmpty() throws Except element.getChild(TAG_DISTANCE_PRODUCT_FILE_PATH).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element 'point_distance-product-file-path' expected", expected.getMessage()); @@ -248,7 +248,7 @@ public void testCreatePostProcessing_DistanceFilePathElementIsMissing() throws E element.removeChild(TAG_DISTANCE_PRODUCT_FILE_PATH); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element 'point_distance-product-file-path' expected", expected.getMessage()); @@ -261,7 +261,7 @@ public void testCreatePostProcessing_LatitudeVarNameTextIsMissing() throws Excep element.getChild(TAG_VAR_NAME_LATITUDE).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_LATITUDE + "' expected", expected.getMessage()); @@ -273,7 +273,7 @@ public void testCreatePostProcessing_LatitudeVarNameTextIsEmpty() throws Excepti element.getChild(TAG_VAR_NAME_LATITUDE).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_LATITUDE + "' expected", expected.getMessage()); @@ -285,7 +285,7 @@ public void testCreatePostProcessing_LatitudeVarNameElementIsMissing() throws Ex element.removeChild(TAG_VAR_NAME_LATITUDE); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element '" + TAG_VAR_NAME_LATITUDE + "' expected", expected.getMessage()); @@ -297,7 +297,7 @@ public void testCreatePostProcessing_LongitudeVarNameTextIsMissing() throws Exce element.getChild(TAG_VAR_NAME_LONGITUDE).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_LONGITUDE + "' expected", expected.getMessage()); @@ -309,7 +309,7 @@ public void testCreatePostProcessing_LongitudeVarNameTextIsEmpty() throws Except element.getChild(TAG_VAR_NAME_LONGITUDE).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_LONGITUDE + "' expected", expected.getMessage()); @@ -321,7 +321,7 @@ public void testCreatePostProcessing_LongitudeVarNameElementIsMissing() throws E element.removeChild(TAG_VAR_NAME_LONGITUDE); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element '" + TAG_VAR_NAME_LONGITUDE + "' expected", expected.getMessage()); @@ -333,7 +333,7 @@ public void testCreatePostProcessing_SourceFileNameVarNameTextIsMissing() throws element.getChild(TAG_VAR_NAME_SOURCE_FILE_NAME).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_SOURCE_FILE_NAME + "' expected", expected.getMessage()); @@ -345,7 +345,7 @@ public void testCreatePostProcessing_SourceFileNameVarNameTextIsEmpty() throws E element.getChild(TAG_VAR_NAME_SOURCE_FILE_NAME).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_SOURCE_FILE_NAME + "' expected", expected.getMessage()); @@ -357,7 +357,7 @@ public void testCreatePostProcessing_SourceFileNameVarNameElementIsMissing() thr element.removeChild(TAG_VAR_NAME_SOURCE_FILE_NAME); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element '" + TAG_VAR_NAME_SOURCE_FILE_NAME + "' expected", expected.getMessage()); @@ -369,7 +369,7 @@ public void testCreatePostProcessing_ProcessingVersionVarNameTextIsMissing() thr element.getChild(TAG_VAR_NAME_PROCESSING_VERSION).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_PROCESSING_VERSION + "' expected", expected.getMessage()); @@ -381,7 +381,7 @@ public void testCreatePostProcessing_ProcessingVersionVarNameTextIsEmpty() throw element.getChild(TAG_VAR_NAME_PROCESSING_VERSION).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_PROCESSING_VERSION + "' expected", expected.getMessage()); @@ -393,7 +393,7 @@ public void testCreatePostProcessing_ProcessingVersionVarNameElementIsMissing() element.removeChild(TAG_VAR_NAME_PROCESSING_VERSION); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element '" + TAG_VAR_NAME_PROCESSING_VERSION + "' expected", expected.getMessage()); @@ -405,7 +405,7 @@ public void testCreatePostProcessing_XVarNameTextIsMissing() throws Exception { element.getChild(TAG_VAR_NAME_SOURCE_X).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_SOURCE_X + "' expected", expected.getMessage()); @@ -417,7 +417,7 @@ public void testCreatePostProcessing_XVarNameTextIsEmpty() throws Exception { element.getChild(TAG_VAR_NAME_SOURCE_X).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_SOURCE_X + "' expected", expected.getMessage()); @@ -429,7 +429,7 @@ public void testCreatePostProcessing_XVarNameElementIsMissing() throws Exception element.removeChild(TAG_VAR_NAME_SOURCE_X); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element '" + TAG_VAR_NAME_SOURCE_X + "' expected", expected.getMessage()); @@ -441,7 +441,7 @@ public void testCreatePostProcessing_YVarNameTextIsMissing() throws Exception { element.getChild(TAG_VAR_NAME_SOURCE_Y).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_SOURCE_Y + "' expected", expected.getMessage()); @@ -453,7 +453,7 @@ public void testCreatePostProcessing_YVarNameTextIsEmpty() throws Exception { element.getChild(TAG_VAR_NAME_SOURCE_Y).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_SOURCE_Y + "' expected", expected.getMessage()); @@ -465,7 +465,7 @@ public void testCreatePostProcessing_processingVersionVarNameElementIsMissing() element.removeChild(TAG_VAR_NAME_SOURCE_Y); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element '" + TAG_VAR_NAME_SOURCE_Y + "' expected", expected.getMessage()); @@ -477,7 +477,7 @@ public void testCreatePostProcessing_SensorNameTextIsMissing() throws Exception element.getChild(TAG_SENSOR_NAME).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_SENSOR_NAME + "' expected", expected.getMessage()); @@ -489,7 +489,7 @@ public void testCreatePostProcessing_SensorNameTextIsEmpty() throws Exception { element.getChild(TAG_SENSOR_NAME).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_SENSOR_NAME + "' expected", expected.getMessage()); @@ -501,7 +501,7 @@ public void testCreatePostProcessing_SensorNameElementIsMissing() throws Excepti element.removeChild(TAG_SENSOR_NAME); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element '" + TAG_SENSOR_NAME + "' expected", expected.getMessage()); @@ -513,7 +513,7 @@ public void testCreatePostProcessing_SourceBtVarNameTextIsMissing() throws Excep element.getChild(TAG_VAR_NAME_SOURCE_BT_11_1_mM).setText(null); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_SOURCE_BT_11_1_mM + "' expected", expected.getMessage()); @@ -525,7 +525,7 @@ public void testCreatePostProcessing_SourceBtVarNameTextIsEmpty() throws Excepti element.getChild(TAG_VAR_NAME_SOURCE_BT_11_1_mM).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element '" + TAG_VAR_NAME_SOURCE_BT_11_1_mM + "' expected", expected.getMessage()); @@ -537,7 +537,7 @@ public void testCreatePostProcessing_SourceBtVarNameElementIsMissing() throws Ex element.removeChild(TAG_VAR_NAME_SOURCE_BT_11_1_mM); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element '" + TAG_VAR_NAME_SOURCE_BT_11_1_mM + "' expected", expected.getMessage()); diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumPluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumPluginTest.java index e20dc2688..a415ca51f 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumPluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumPluginTest.java @@ -21,8 +21,8 @@ package com.bc.fiduceo.post.plugin.iasi; import com.bc.fiduceo.post.PostProcessing; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; @@ -49,7 +49,7 @@ public void testGetPostProcessingName() { public void testCreatePostProcessing() throws JDOMException, IOException { final Element rootElement = AddIASISpectrumTest.createFullConfigElement(); - final PostProcessing postProcessing = plugin.createPostProcessing(rootElement); + final PostProcessing postProcessing = plugin.createPostProcessing(rootElement, null); assertTrue(postProcessing instanceof AddIASISpectrum); } } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumTest.java index cf46e0221..abb399136 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/iasi/AddIASISpectrumTest.java @@ -26,8 +26,8 @@ import com.bc.fiduceo.reader.iasi.IASI_Reader; import com.bc.fiduceo.util.NetCDFUtils; import org.hamcrest.CoreMatchers; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; import ucar.ma2.Array; diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistancePluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistancePluginTest.java index a5c535926..4f6fb47ba 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistancePluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistancePluginTest.java @@ -2,8 +2,8 @@ import com.bc.fiduceo.post.PostProcessing; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; @@ -30,7 +30,7 @@ public void testGetPostProcessingName() { public void testCreatePostProcessing() throws JDOMException, IOException { final Element configElement = AddLandDistanceTest.createFullConfigElement(); - final PostProcessing postProcessing = plugin.createPostProcessing(configElement); + final PostProcessing postProcessing = plugin.createPostProcessing(configElement, null); assertTrue(postProcessing instanceof AddLandDistance); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistanceTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistanceTest.java index cb048df13..79ef594c7 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistanceTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/land_distance/AddLandDistanceTest.java @@ -5,8 +5,8 @@ import com.bc.fiduceo.TestUtil; import com.bc.fiduceo.post.util.DistanceToLandMap; import com.bc.fiduceo.util.NetCDFUtils; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Test; import ucar.ma2.Array; import ucar.ma2.DataType; diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPluginTest.java index 53535bfc8..bcdfc2ddf 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPluginTest.java @@ -22,8 +22,8 @@ import com.bc.fiduceo.TestUtil; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.Before; import org.junit.Test; diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPlugin_IO_Test.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPlugin_IO_Test.java index 3676bfd45..7fa194a16 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPlugin_IO_Test.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/nwp/NwpPostProcessingPlugin_IO_Test.java @@ -24,8 +24,8 @@ import com.bc.fiduceo.IOTestRunner; import com.bc.fiduceo.TestUtil; import com.bc.fiduceo.post.PostProcessing; -import org.jdom.Element; -import org.jdom.JDOMException; +import org.jdom2.Element; +import org.jdom2.JDOMException; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -73,7 +73,7 @@ public void testCreatePostProcessing() throws JDOMException, IOException { ""; final Element rootElement = TestUtil.createDomElement(config); - final PostProcessing postProcessing = plugin.createPostProcessing(rootElement); + final PostProcessing postProcessing = plugin.createPostProcessing(rootElement, null); assertNotNull(postProcessing); assertTrue(postProcessing instanceof NwpPostProcessing); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/point_distance/SphericalDistancePluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/point_distance/SphericalDistancePluginTest.java index ca5220083..6f99628b0 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/point_distance/SphericalDistancePluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/point_distance/SphericalDistancePluginTest.java @@ -23,7 +23,7 @@ import static org.junit.Assert.*; import com.bc.fiduceo.post.PostProcessing; -import org.jdom.Element; +import org.jdom2.Element; import org.junit.*; import java.util.Arrays; @@ -50,7 +50,7 @@ public void setUp() throws Exception { @Test public void testCreatePostProcessing() { - final PostProcessing postProcessing = plugin.createPostProcessing(element); + final PostProcessing postProcessing = plugin.createPostProcessing(element, null); assertNotNull(postProcessing); assertEquals("com.bc.fiduceo.post.plugin.point_distance.SphericalDistance", postProcessing.getClass().getTypeName()); @@ -78,7 +78,7 @@ public void testCreatePostProcessing_wrongElementName() throws Exception { element.setName("wrong_name"); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Illegal XML Element. Tagname 'spherical-distance' expected.", expected.getMessage()); @@ -90,7 +90,7 @@ public void testCreatePostProcessing_targetElementIsMissing() throws Exception { element.removeChild(TAG_NAME_TARGET); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element 'target' expected", expected.getMessage()); @@ -102,7 +102,7 @@ public void testCreatePostProcessing_targetVarNameElementIsMissing() throws Exce element.getChild(TAG_NAME_TARGET).removeChild(TAG_NAME_VAR_NAME); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element 'var-name' expected", expected.getMessage()); @@ -114,7 +114,7 @@ public void testCreatePostProcessing_targetVarNameElement_valueIsMissing() throw element.getChild(TAG_NAME_TARGET).getChild(TAG_NAME_VAR_NAME).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element 'var-name' expected", expected.getMessage()); @@ -126,7 +126,7 @@ public void testCreatePostProcessing_targetDataTypeElementIsMissing() throws Exc element.getChild(TAG_NAME_TARGET).removeChild(TAG_NAME_DATA_TYPE); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element 'data-type' expected", expected.getMessage()); @@ -138,7 +138,7 @@ public void testCreatePostProcessing_targetDataTypeElement_valueIsMissing() thro element.getChild(TAG_NAME_TARGET).getChild(TAG_NAME_DATA_TYPE).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element 'data-type' expected", expected.getMessage()); @@ -150,7 +150,7 @@ public void testCreatePostProcessing_primaryLatElemElementIsMissing() throws Exc element.removeChild(TAG_NAME_PRIM_LAT_VAR); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element 'primary-lat-variable' expected", expected.getMessage()); @@ -162,7 +162,7 @@ public void testCreatePostProcessing_primaryLatElemElement_valueIsMissing() thro element.getChild(TAG_NAME_PRIM_LAT_VAR).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element 'primary-lat-variable' expected", expected.getMessage()); @@ -174,7 +174,7 @@ public void testCreatePostProcessing_primaryLonElemElementIsMissing() throws Exc element.removeChild(TAG_NAME_PRIM_LON_VAR); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element 'primary-lon-variable' expected", expected.getMessage()); @@ -186,7 +186,7 @@ public void testCreatePostProcessing_primaryLonElemElement_valueIsMissing() thro element.getChild(TAG_NAME_PRIM_LON_VAR).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element 'primary-lon-variable' expected", expected.getMessage()); @@ -198,7 +198,7 @@ public void testCreatePostProcessing_secundaryLatElemElementIsMissing() throws E element.removeChild(TAG_NAME_SECO_LAT_VAR); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element 'secondary-lat-variable' expected", expected.getMessage()); @@ -210,7 +210,7 @@ public void testCreatePostProcessing_secundaryLatElemElement_valueIsMissing() th element.getChild(TAG_NAME_SECO_LAT_VAR).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element 'secondary-lat-variable' expected", expected.getMessage()); @@ -222,7 +222,7 @@ public void testCreatePostProcessing_secundaryLonElemElementIsMissing() throws E element.removeChild(TAG_NAME_SECO_LON_VAR); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Child element 'secondary-lon-variable' expected", expected.getMessage()); @@ -234,7 +234,7 @@ public void testCreatePostProcessing_secundaryLonElemElement_valueIsMissing() th element.getChild(TAG_NAME_SECO_LON_VAR).setText(""); try { - plugin.createPostProcessing(element); + plugin.createPostProcessing(element, null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertEquals("Value of element 'secondary-lon-variable' expected", expected.getMessage()); diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesPluginTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesPluginTest.java index 4859e4d96..4396c1507 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesPluginTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesPluginTest.java @@ -21,20 +21,13 @@ import com.bc.fiduceo.post.PostProcessing; import com.bc.fiduceo.post.PostProcessingPlugin; -import org.jdom.Element; +import org.jdom2.Element; import org.junit.Before; import org.junit.Test; import java.util.Arrays; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_FILE_NAME_VARIABLE; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_INSITU_SENSOR; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_SECONDARY_SENSOR_MATCHUP_TIME_VARIABLE; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_SST_INSITU_TIME_SERIES; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_TIME_RANGE_SECONDS; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_TIME_SERIES_SIZE; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_VERSION; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_Y_VARIABLE; +import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.*; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; @@ -57,6 +50,7 @@ public void setUp() { new Element(TAG_NAME_VERSION).addContent("v03.3"), new Element(TAG_NAME_TIME_RANGE_SECONDS).addContent("" + 36 * 60 * 60), new Element(TAG_NAME_TIME_SERIES_SIZE).addContent("96"), + new Element(TAG_NAME_INSITU_TIME_VARIABLE).addContent("drifter-sirds.acquisition_time"), new Element(TAG_NAME_SECONDARY_SENSOR_MATCHUP_TIME_VARIABLE).addContent("amsre.acquisition_time") )); } @@ -68,7 +62,7 @@ public void testThatPluginImplementsInterfacePostProcessingPlugin() { @Test public void testThatPluginReturnsASstInsituTimeSeriesPostProcessing() { - final PostProcessing postProcessing = plugin.createPostProcessing(element); + final PostProcessing postProcessing = plugin.createPostProcessing(element, null); assertThat(postProcessing, is(not(nullValue()))); assertThat(postProcessing, instanceOf(PostProcessing.class)); @@ -78,7 +72,7 @@ public void testThatPluginReturnsASstInsituTimeSeriesPostProcessing() { @Test public void createPostProcessing_throwsExceptionIfTheNameOfTheRootElementIsWrong() { try { - plugin.createPostProcessing(new Element("wrongName")); + plugin.createPostProcessing(new Element("wrongName"), null); fail("RuntimeException expected"); } catch (RuntimeException expected) { assertThat(expected.getMessage(), is(equalTo("Illegal XML Element. Tagname '" + plugin.getPostProcessingName() + "' expected."))); @@ -92,6 +86,7 @@ public void testParseConfiguration() { assertEquals(36 * 60 * 60, configuration.timeRangeSeconds); assertEquals(96, configuration.timeSeriesSize); assertEquals("amsre.acquisition_time", configuration.matchupTimeVarName); + assertEquals("drifter-sirds.acquisition_time", configuration.insituTimeVarName); assertNull(configuration.insituSensorName); } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesTest.java index 0b9cbe490..166364363 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesTest.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeriesTest.java @@ -203,4 +203,15 @@ public void testAddInsituVariables() throws Exception { verify(newVar, times(3)).addAttribute(any(Attribute.class)); verifyNoMoreInteractions(writer, insituReader, newVar, v3, v2); } + + @Test + public void testFileNameMatches() { + assertTrue(SstInsituTimeSeries.nameMatches("insitu_0_WMOID_42531_19960904_19960909.nc")); + assertTrue(SstInsituTimeSeries.nameMatches("insitu_0_WMOID_46942_19951026_19951027.nc")); + assertTrue(SstInsituTimeSeries.nameMatches("SSTCCI2_refdata_gtmba2_201204.nc")); + assertTrue(SstInsituTimeSeries.nameMatches("SSTCCI2_refdata_drifter_201701.nc")); + + assertFalse(SstInsituTimeSeries.nameMatches("PIRATA_0N35W_sss_2016-10.txt")); + assertFalse(SstInsituTimeSeries.nameMatches("ASCAT-vs-AMSR2-vs-ERA5-vs-DMISIC0-2016-N.text")); + } } diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeries_IO_Test.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeries_IO_Test.java index 5a2b1f6fb..dd699da31 100644 --- a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeries_IO_Test.java +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/sstInsitu/SstInsituTimeSeries_IO_Test.java @@ -33,10 +33,10 @@ import com.bc.fiduceo.reader.insitu.sst_cci.SSTInsituReader; import com.bc.fiduceo.util.NetCDFUtils; import org.esa.snap.core.util.io.FileUtils; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.output.Format; -import org.jdom.output.XMLOutputter; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.output.Format; +import org.jdom2.output.XMLOutputter; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -56,11 +56,7 @@ import java.nio.file.Path; import java.util.Arrays; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_SECONDARY_SENSOR_MATCHUP_TIME_VARIABLE; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_SST_INSITU_TIME_SERIES; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_TIME_RANGE_SECONDS; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_TIME_SERIES_SIZE; -import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.TAG_NAME_VERSION; +import static com.bc.fiduceo.post.plugin.sstInsitu.SstInsituTimeSeriesPlugin.*; import static com.bc.fiduceo.util.NetCDFUtils.CF_FILL_VALUE_NAME; import static com.bc.fiduceo.util.NetCDFUtils.CF_LONG_NAME; import static com.bc.fiduceo.util.NetCDFUtils.CF_UNITS_NAME; @@ -144,6 +140,7 @@ public void prepare() throws Exception { configuration.timeRangeSeconds = 124; configuration.timeSeriesSize = 16; configuration.matchupTimeVarName = "matchupTimeVarName"; + configuration.insituTimeVarName = "insituTimeVarName"; final SstInsituTimeSeries insituTimeSeries = new SstInsituTimeSeries(configuration); insituTimeSeries.setContext(context); @@ -208,6 +205,7 @@ public void computeInsituRange() throws Exception { configuration.timeRangeSeconds = seconds; configuration.timeSeriesSize = 300; configuration.matchupTimeVarName = "matchupTimeVarName"; + configuration.insituTimeVarName = "insitu.time"; final SstInsituTimeSeries insituTimeSeries = new SstInsituTimeSeries(configuration); final ReaderFactory readerFactory = ReaderFactory.create(new GeometryFactory("S2"), null, null, null); // we don't need temp file support here tb 2018-01-23 @@ -260,6 +258,7 @@ public void computeOneProduct() throws Exception { new Element(TAG_NAME_VERSION).addContent("v03.3"), new Element(TAG_NAME_TIME_RANGE_SECONDS).addContent("" + 80000), new Element(TAG_NAME_TIME_SERIES_SIZE).addContent("10"), + new Element(TAG_NAME_INSITU_TIME_VARIABLE).addContent("insitu.time"), new Element(TAG_NAME_SECONDARY_SENSOR_MATCHUP_TIME_VARIABLE).addContent("amsre.acquisition_time") )) ) diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/util/PPUtilsTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/util/PPUtilsTest.java new file mode 100644 index 000000000..41222a2b2 --- /dev/null +++ b/post-processing-tool/src/test/java/com/bc/fiduceo/post/util/PPUtilsTest.java @@ -0,0 +1,49 @@ +package com.bc.fiduceo.post.util; + +import org.junit.Test; +import ucar.ma2.Array; + +import static org.hamcrest.Matchers.*; +import static org.hamcrest.MatcherAssert.*; +import static org.junit.Assert.*; + +public class PPUtilsTest { + + @Test + public void convertToFitTheRangeMinus180to180() { + final int farOutside = 360 * 10; + + final double[] lons = { + -180, + -179.999999999, + 179.999999999, + 180, + -190, + 190, + -30 - farOutside, + 40 + farOutside, + Double.NaN, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY + }; + + final Array lonArray = Array.makeFromJavaArray(lons); + PPUtils.convertToFitTheRangeMinus180to180(lonArray); + + final double[] expected = { + /* Before: -180 after -> */ -180, + /* Before: -179.999999999 after -> */ -179.999999999, + /* Before: 179.999999999 after -> */ 179.999999999, + /* Before: 180 after -> */ 180, + /* Before: -190 after -> */ 170, + /* Before: 190 after -> */ -170, + /* Before: -30 - farOutside after -> */ -30, + /* Before: 40 + farOutside after -> */ 40, + /* Before: Double.NaN after -> */ Double.NaN, + /* Before: Double.NEGATIVE_INFINITY after -> */ Double.NEGATIVE_INFINITY, + /* Before: Double.POSITIVE_INFINITY after -> */ Double.POSITIVE_INFINITY + }; + + assertArrayEquals(expected, (double[]) lonArray.getStorage(), 0.000001); + } +} \ No newline at end of file