GFZ-Centre-for-Early-Warning · ruester · Sep 23, 2019 · Sep 23, 2019 · Sep 23, 2019 · Sep 27, 2019
diff --git a/.gitignore b/.gitignore
@@ -36,7 +36,7 @@ pip-log.txt
 pip-delete-this-directory.txt
 
 # Unit test / coverage reports
-htmlcov/
+htmlcov/schema
 .tox/
 .coverage
 .coverage.*
@@ -49,7 +49,7 @@ coverage.xml
 
 # Translations
 *.mo
-*.pot
+*.potschema
 
 # Django stuff:
 *.log
@@ -102,3 +102,10 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+*.swp
+
+# Mac OS stuff
+.DS_Store
+
+.idea
+output/*
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,154 @@
+# Copyright © 2021 Helmholtz Centre Potsdam GFZ German Research Centre for Geosciences, Potsdam, Germany
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+stages:
+  - test
+  - build
+  - deploy
+
+cache:
+  key: "$CI_COMMIT_REF_SLUG"
+  paths:
+    - .pip
+
+.install_requirements: &install_requirements
+  - DEBIAN_FRONTEND=noninteractive apt-get update && apt-get upgrade -y
+  - pip3 install --cache-dir=.pip -r requirements.txt
+
+verify_python:
+  image: python:3.7-buster
+  stage: test
+  tags:
+    - group
+  before_script:
+    - *install_requirements
+  script:
+    - python3 -m compileall .
+
+python_pycodestyle:
+  image: python:3.7-buster
+  stage: test
+  # TODO: fix code to fit with PEP8 guidelines
+  allow_failure: true
+  tags:
+    - group
+  before_script:
+    - *install_requirements
+    - pip3 install --cache-dir=.pip pycodestyle
+  script:
+    - shopt -s globstar && pycodestyle {,**/}*.py
+
+python_pylint:
+  image: python:3.7-buster
+  stage: test
+  # TODO: fix code to get a better pylint rating
+  allow_failure: true
+  tags:
+    - group
+  before_script:
+    - *install_requirements
+    - pip3 install --cache-dir=.pip pylint
+  script:
+    - shopt -s globstar && pylint --fail-under=8 {,**/}*.py
+
+python_black:
+  image: python:3.7-buster
+  stage: test
+  tags:
+    - group
+  before_script:
+    - *install_requirements
+    - pip3 install --cache-dir=.pip black==22.3.0
+  script:
+    - black -l 79 --check .
+
+run_tests:
+  image: python:3.7-buster
+  stage: test
+  tags:
+    - group
+  before_script:
+    - *install_requirements
+  script:
+    - python3 test_all.py
+
+run_notebook:
+  image: python:3.7-buster
+  stage: test
+  tags:
+    - group
+  before_script:
+    - *install_requirements
+    - pip3 install scipy jupyter nbconvert ipykernel matplotlib
+  script:
+    - jupyter nbconvert --to html --execute --ExecutePreprocessor.kernel_name=python3 --output output.ipynb service_test.ipynb
+
+verify_json:
+  stage: test
+  image: registry.gitlab.com/pipeline-components/jsonlint:0.12.0
+  cache: {}
+  script:
+    - |
+      find . -not -path './.git/*' -name '*.json' -type f -print0 |
+      parallel --will-cite -k -0 -n1 jsonlint -q
+
+dockerbuild_latest:
+  image: docker/compose:1.29.2
+  stage: build
+  cache: {}
+  tags:
+    - group
+  script:
+    - docker build -t modelprop:latest -f metadata/Dockerfile .
+
+    # push latest image to Docker Hub
+    - docker tag modelprop:latest gfzriesgos/modelprop:latest
+    - docker login -u ${DOCKERHUB_USER} -p ${DOCKERHUB_PASSWORD}
+    - docker push gfzriesgos/modelprop:latest
+
+    # push latest image to GitLab Container Registry
+    - docker tag modelprop:latest git.gfz-potsdam.de:5000/id2/riesgos/modelprop:latest
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker push git.gfz-potsdam.de:5000/id2/riesgos/modelprop:latest
+
+dockerbuild_tagged:
+  image: docker/compose:1.29.2
+  stage: build
+  cache: {}
+  tags:
+    - group
+  rules:
+    - if: $CI_COMMIT_TAG
+  script:
+    - sed -i -e "s@:latest@:${CI_COMMIT_TAG}@g" metadata/modelprop.json
+    - docker build -t modelprop:${CI_COMMIT_TAG} -f metadata/Dockerfile .
+
+    # push image to Docker Hub
+    - docker tag modelprop:${CI_COMMIT_TAG} gfzriesgos/modelprop:${CI_COMMIT_TAG}
+    - docker login -u ${DOCKERHUB_USER} -p ${DOCKERHUB_PASSWORD}
+    - docker push gfzriesgos/modelprop:${CI_COMMIT_TAG}
+
+    # push image to GitLab Container Registry
+    - docker tag modelprop:${CI_COMMIT_TAG} git.gfz-potsdam.de:5000/id2/riesgos/modelprop:${CI_COMMIT_TAG}
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker push git.gfz-potsdam.de:5000/id2/riesgos/modelprop:${CI_COMMIT_TAG}
+
+deploy_latest:
+  stage: deploy
+  variables:
+    DEPLOY_LATEST_IMAGES: "true"
+  trigger:
+    project: id2/riesgos/gfz-riesgos-wps-repository
+    branch: master
+    strategy: depend
diff --git a/AUTHORS.md b/AUTHORS.md
@@ -0,0 +1,5 @@
+Max Pittore <pittore@gfz-potsdam.de>
+Nils Brinckmann <nils.brinckmann@gfz-potsdam.de>
+Simantini Shinde <shinde@gfz-potsdam.de>
+Matthias Rüster <matthias.ruester@gfz-potsdam.de>
+Juan Camilo Gomez <jcgomez@gfz-potsdam.de>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,17 @@
+# 2019-09-23
+
+- Added the changelog
+- Added the schemas for hazus and suppasri
+- Updated the schema for sara
+- Updated the code for finding supported schemas
+- Updated json configuration for gfz riesgos wps
+  integration for supported schemas
+- Updated README.md
+- Added first unittest
+- Code is now pep8 conform
+- Code is now mostly pylint3 conform
+
+# 2019-06-16
+
+- Last version purely by Max Pittore (added the metadata for
+  the gfz riesgos wps integration)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -0,0 +1,36 @@
+# Contributing to the wps-pipeline-notebook
+
+Please feel free to contribute to modelprop.
+You can fork it and adopt it as you need, but we would like to
+have also extensions flowing back into the original code base.
+
+## Version schema
+We use a version schema that is just date oriented.
+
+## Branching policy
+We use a development branch for every feature that we add.
+Master branch should just be used for
+a) the initial version
+b) the place to merge feature branches into.
+
+(As this code is written initially by a single person, most of the stuff
+is done actually within the master branch).
+
+
+## Testing
+We use unittests.
+Please make sure at least, that you write tests for the code you add 
+or change.
+
+## Code styling
+
+For code styling we use pep8 and pylint3.
+Make sure that your code is at least compliant with pep8.
+
+There is no need to care about pylint3 warnings regarding to
+too less public methods in classes. Those are a good indicator
+to use functions / partitial functions instead, but using classes
+is clearer in most cases.
+
+Also we don't care about unused arguments once methods implement a
+common protocol and those variables are used in other implementations.
diff --git a/LICENSE b/LICENSE
@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright [2021] [Helmholtz Centre Potsdam GFZ German Research Centre for Geosciences]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

diff --git a/README.md b/README.md
@@ -1,2 +1,96 @@
 # modelprop
-Program to serve a fragility / vulnerability model according to a given schema - RIESGOS Project
+
+
+[![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/gfzriesgos/modelprop)](https://hub.docker.com/r/gfzriesgos/modelprop)
+[![Build Status](https://travis-ci.com/gfzriesgos/modelprop.svg?branch=master)](https://travis-ci.com/gfzriesgos/modelprop)
+
+## Purpose
+Program to serve a fragility / vulnerability model
+according to a given schema - RIESGOS Project
+
+## Citation
+
+Pittore, Massimiliano; Gomez-Zapata, Juan Camilo; Brinckmann, Nils; Rüster, Matthias (2021): Assetmaster and Modelprop: web services to serve building exposure models and fragility functions for physical vulnerability to natural-hazards. V. 1.0. GFZ Data Services. https://doi.org/10.5880/riesgos.2021.005
+
+## Setup
+
+We use a virtual environment, so you can need to create it and
+install the pacakges from pip:
+
+```shell
+python3 -m venv env
+source env/bin/activate
+pip install -r requirements.txt
+```
+
+## Example run
+After that you can run modelprop as in the following example:
+
+```shell
+python3 modelprop.py SARA-v1.0 buildings structural
+```
+
+What are the parameters:
+
+| Parameter      | Example    | Purpose                                                                                            |
+|----------------|------------|----------------------------------------------------------------------------------------------------|
+| schema         | SARA-v1.0  | This is the schema for the taxonomy.                                                               |
+| asset category | buildings  | In this case we are only interested in buildings, and not in other infrastructure.                 |
+| loss category  | structural | In this case we care about physical damage on the structure of the buildings.                      |
+| taxonomy       |            | You can use this to just query a subset of the schema giving a taxonomy. This is empty by default. |
+
+## Supported schemas
+
+At the moment we support the following schemas:
+* HAZUS-v1.0
+* SARA-v1.0
+* SUPPASRI2013-v2.0
+* Mavrouli et al. 2014
+* Torres Corredor et al. 2017
+
+You can get a up to date list of the supported schemas by running the following commands
+in a python shell from within this folder:
+
+```python
+import modelprop
+modelprop.get_supported_schemas()
+```
+
+
+## Supported asset categories
+
+At the moment only buildings are supported.
+
+## Supported loss categories
+
+At the moment we only support structural as loss category.
+
+## How to add new schemas
+
+### Naming and paths
+
+The very first part is to check that the name of your schema file has the name `<your_schema_name>_struct.json` and
+that this will be inserted in the folder `<your_schema_name>` in the schema subfolder.
+
+### Data
+It is important to have a meta block, with an id and the list of taxonomies and a data list with the
+fragility function data for your taxonomy.
+
+Every taxonomy that is listed in the data list must also be in the meta section and vice versa.
+
+You can test it with the `assistance/check_modelprop_json_file.py` file.
+
+## Copyright & License
+Copyright © 2021 Helmholtz Centre Potsdam GFZ German Research Centre for Geosciences, Potsdam, Germany
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+
+## Project
+
+Deus was developed in the scope of the RIESGOS project:
+Multi-Risk Analysis and Information System Components for the Andes Region (https://www.riesgos.de/en)
+
diff --git a/__init__.py b/__init__.py
@@ -0,0 +1,7 @@
+# Copyright © 2021 Helmholtz Centre Potsdam GFZ German Research Centre for Geosciences, Potsdam, Germany
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.