CODAL has been building with `-O2` flag instead of `-Os`

[microbit-carlos]

If we build with the verbose output we can see it at the end of each file compilation:

$ python build.py --verbose

...
[ 89%] Building CXX object CMakeFiles/MICROBIT.dir/source/main.cpp.obj
/Users/microbit-carlos/opt/gcc-arm-none-eabi-10.3-2021.10/bin/arm-none-eabi-g++ -DMICROBIT_EXPORTS -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/source/samples" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-v2/inc" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-v2/inc/bluetooth" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-v2/inc/compat" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-v2/model" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-nrf52/inc" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-nrf52/inc/cmsis" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-nrf52/nrfx/mdk" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-nrf52/nrfx" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-nrf52/nrfx/templates" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-nrf52/nrfx/templates/nRF52833" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-nrf52/nrfx/drivers/include" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-nrf52/nrfx/drivers/src" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-core/./inc/core" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-core/./inc/driver-models" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-core/./inc/drivers" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-core/./inc/streams" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-core/./inc/types" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/../codal-nrf52/nrfx/hal" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/sdk_config" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK_mods" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/pwr_mgmt" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/sortlist" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/strerror" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/softdevice/common" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/crc32" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/bootloader/dfu" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/util" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/ble/common" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/balloc" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/ble/peer_manager" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/ringbuf" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/timer" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/log" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/ble/nrf_ble_gatt" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/bootloader" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/fstorage" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/experimental_section_vars" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/softdevice/s113/headers" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/mutex" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/delay" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/bootloader/ble_dfu" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/atomic_fifo" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/atomic" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/memobj" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/fds" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/atomic_flags" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/softdevice/s113/headers/nrf52" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/ble/ble_services/ble_dfu" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/external/fprintf" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/svc" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/libraries/log/src" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/libraries/codal-microbit-nrf5sdk/nRF5SDK/components/ble/ble_services/ble_dis" -Wno-expansion-to-defined -mcpu=cortex-m4 -mthumb -mfpu=fpv4-sp-d16 -mfloat-abi=softfp -fno-exceptions -fno-unwind-tables -ffunction-sections -fdata-sections -Wall -Wextra -Wno-unused-parameter -std=c++11 -fwrapv -fno-rtti -fno-threadsafe-statics -fno-exceptions -fno-unwind-tables -Wl,--gc-sections -Wl,--sort-common -Wl,--sort-section=alignment -Wno-array-bounds -include "/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/build/codal_extra_definitions.h" -I"/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/utils/cmake/toolchains/ARM_GCC" -O2 -g -DNDEBUG   -DAPP_TIMER_V2 -DAPP_TIMER_V2_RTC1_ENABLED -DNRF_DFU_TRANSPORT_BLE=1 -DNRF52833_XXAA -DNRF52833 -DTARGET_MCU_NRF52833 -DNRF5 -DNRF52833 -D__CORTEX_M4 -DS113 -DTOOLCHAIN_GCC -D__START=target_start -MMD -MT CMakeFiles/MICROBIT.dir/source/main.cpp.obj -MF DEPFILE -o CMakeFiles/MICROBIT.dir/source/main.cpp.obj -c "/Users/microbit-carlos/workspace/tmp/untitled folder/microbit-v2-samples/source/main.cpp"

The interesting part is this bit:

-O2 -g -DNDEBUG

Which I believe is the CMake flags configured for the RelWithDebInfo CMake build type (the default). To test this, we can add these messages at the top of the CMakefile.txt file and see the result:

message("!!!! CMAKE_BUILD_TYPE is ${CMAKE_BUILD_TYPE}")
message("!!!! CMAKE_CXX_FLAGS_DEBUG default value is ${CMAKE_CXX_FLAGS_DEBUG}")
message("!!!! CMAKE_CXX_FLAGS_RELEASE default value is ${CMAKE_CXX_FLAGS_RELEASE}")
message("!!!! CMAKE_CXX_FLAGS_RELWITHDEBINFO default value is ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
message("!!!! CMAKE_CXX_FLAGS_MINSIZEREL default value is ${CMAKE_CXX_FLAGS_MINSIZEREL}")

Output:

!!!! CMAKE_BUILD_TYPE is RelWithDebInfo
!!!! CMAKE_CXX_FLAGS_DEBUG default value is -g
!!!! CMAKE_CXX_FLAGS_RELEASE default value is -O3 -DNDEBUG
!!!! CMAKE_CXX_FLAGS_RELWITHDEBINFO default value is -O2 -g -DNDEBUG
!!!! CMAKE_CXX_FLAGS_MINSIZEREL default value is -Os -DNDEBUG

The microbit-v2-samples repo is meant to be setting this value with CMAKE_CXX_FLAGS_RELWITHDEBINFO_INIT:
lancaster-university/microbit-v2-samples/utils/cmake/toolchains/ARM_GCC/compiler-flags.cmake#L42

set(CMAKE_CXX_FLAGS_RELWITHDEBINFO_INIT "-Os -g -gdwarf-3 -DNDEBUG")

But we have two issues:

• CMAKE_<lang>_FLAGS_<config>_INIT was only introduced in CMake 3.11, and we support 3.6+, so in those versions CMAKE_CXX_FLAGS_RELWITHDEBINFO_INIT is not doing anything
  • https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_FLAGS_CONFIG_INIT.html
• It doesn't seem to be working anyway, as proven by this bug report
  • Also mentioned by Rob Moran in this comment that changing the values in that variable didn't work Configure debug info to DWARF v4. microbit-v2-samples#82 (comment)

So we should probably explicitly set the build type to RelWithDebInfo (while still being configurable via CMake flag) and correctly configure optimisation flags by using CMAKE_<lang>_FLAGS_<config> instead of CMAKE_<lang>_FLAGS_<config>_INIT, but the question might be:

Do we want to move to -Os, if it might change the memory footprint?

Usually the tradeoffs for flash might be performance (less inlining and things like that), but is it possible it might increase RAM usage at this stage?

[microbit-carlos]

The -O2 flag can be seen in CI as well simply by building with the --verbose flag as well:

• Test commit: lancaster-university/microbit-v2-samples@743c10d
• CI output: https://github.com/lancaster-university/microbit-v2-samples/actions/runs/6124803101/job/16625497443
  • The output is too large and it gets truncated, but you can open the "raw output" and search for -O2
  • https://pipelinesghubeus2.actions.githubusercontent.com/Iad94guzzWXjgmgu46bsCnSrlOSxekZ69pabbjboJPQCMSMx7u/_apis/pipelines/1/runs/498/signedlogcontent/10?urlExpires=2023-09-08T17%3A54%3A02.7637079Z&urlSigningMethod=HMACV1&urlSignature=KlWgvEJZzh%2BZQtQJYB66%2Fl1s4VP5di1uNd2dvelWdQU%3D

[microbit-carlos]

Changing for and using bloaty to checkout the size diff, we might be able to get 13 KBs of flash:

$ docker run --rm -v $(pwd):/home ghcr.io/carlosperate/bloaty -d sections --domain=vm -s vm -n 0 build/MICROBIT -- build-02/MICROBIT
     VM SIZE
 --------------
  -1.2%      -4    .data
 -10.9% -13.3Ki    .text
  -3.5% -13.3Ki    TOTAL

$ docker run --rm -v $(pwd):/home ghcr.io/carlosperate/bloaty -d compileunits --domain=vm -s vm -n 0 build/MICROBIT -- build-02/MICROBIT
     VM SIZE
 --------------
 +20e2% +2.75Ki    ./microbit-v2-samples/source/samples/OOB.cpp
  +1.4%     +76    ./microbit-v2-samples/source/main.cpp
   +28%     +20    ./microbit-v2-samples/libraries/codal-microbit-v2/source/bluetooth/MicroBitBLEServices.cpp
   +16%     +18    ./microbit-v2-samples/libraries/codal-microbit-v2/source/PacketBuffer.cpp
  +1.4%      +4    ./microbit-v2-samples/libraries/codal-core/source/streams/MemorySource.cpp
  +3.0%      +2    [section .bss]
  -0.3%      -2    ./microbit-v2-samples/libraries/codal-core/source/streams/StreamNormalizer.cpp
  -1.2%      -2    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitCompass.cpp
  -2.5%      -4    [section .data]
  -0.5%      -6    ./microbit-v2-samples/libraries/codal-core/source/streams/Synthesizer.cpp
 -11.5%      -6    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitCompat.cpp
  -1.7%      -6    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitLog.cpp
  -7.7%      -6    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitRadioDatagram.cpp
  -0.6%      -8    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitAudio.cpp
  -3.3%      -8    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitThermometer.cpp
  -1.3%      -8    ./microbit-v2-samples/libraries/codal-nrf52/source/peripheral_alloc.cpp
  [DEL]      -8    ./microbit-v2-samples/source/samples/MicrophoneTest.cpp
  -7.7%     -10    ./microbit-v2-samples/libraries/codal-core/source/types/RefCounted.cpp
 -19.2%     -10    ./microbit-v2-samples/libraries/codal-microbit-v2/source/FSCache.cpp
  -5.2%     -12    ./microbit-v2-samples/libraries/codal-core/source/core/CodalCompat.cpp
 -14.0%     -12    ./microbit-v2-samples/libraries/codal-core/source/core/MemberFunctionCallback.cpp
 -13.5%     -14    ./microbit-v2-samples/libraries/codal-core/source/driver-models/AbstractButton.cpp
  -2.9%     -14    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitAccelerometer.cpp
  -5.9%     -16    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitFlash.cpp
  -1.6%     -16    ./microbit-v2-samples/libraries/codal-microbit-v2/source/SoundSynthesizerEffects.cpp
  -4.3%     -18    ./microbit-v2-samples/libraries/codal-core/source/types/ManagedBuffer.cpp
 -10.7%     -18    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitDisplay.cpp
 -13.1%     -28    ./microbit-v2-samples/libraries/codal-core/source/core/CodalListener.cpp
 -15.5%     -32    ./microbit-v2-samples/libraries/codal-core/source/driver-models/Display.cpp
  -3.2%     -32    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitRadio.cpp
  -7.9%     -34    ./microbit-v2-samples/libraries/codal-nrf52/source/NRF52TouchSensor.cpp
 -37.3%     -38    ./microbit-v2-samples/libraries/codal-core/source/driver-models/PinPeripheral.cpp
  -6.5%     -38    ./microbit-v2-samples/libraries/codal-core/source/streams/StreamRecording.cpp
 -17.8%     -38    ./microbit-v2-samples/libraries/codal-core/source/types/CoordinateSystem.cpp
  -9.6%     -38    ./microbit-v2-samples/libraries/codal-core/source/types/ManagedString.cpp
  -5.2%     -40    ./microbit-v2-samples/libraries/codal-core/source/streams/DataStream.cpp
 -16.7%     -44    ./microbit-v2-samples/libraries/codal-microbit-v2/source/bluetooth/MicroBitPartialFlashingService.cpp
  -6.5%     -46    ./microbit-v2-samples/libraries/codal-core/source/drivers/LSM303Accelerometer.cpp
  -2.5%     -46    ./microbit-v2-samples/libraries/codal-microbit-v2/model/MicroBit.cpp
  -9.5%     -48    ./microbit-v2-samples/libraries/codal-core/source/drivers/LSM303Magnetometer.cpp
 -12.3%     -52    ./microbit-v2-samples/libraries/codal-microbit-v2/source/NRF52FlashManager.cpp
 -14.3%     -58    ./microbit-v2-samples/libraries/codal-core/source/core/CodalHeapAllocator.cpp
 -13.3%     -58    ./microbit-v2-samples/libraries/codal-core/source/driver-models/Compass.cpp
  -9.2%     -60    ./microbit-v2-samples/libraries/codal-core/source/drivers/TouchSensor.cpp
  -6.6%     -64    ./microbit-v2-samples/libraries/codal-core/source/drivers/MultiButton.cpp
  -6.3%     -86    ./microbit-v2-samples/libraries/codal-microbit-v2/source/bluetooth/MicroBitBLEManager.cpp
  -4.6%     -96    ./microbit-v2-samples/source/samples/OOB_v3.cpp
 -58.1%    -100    ./microbit-v2-samples/libraries/codal-core/source/core/CodalUtil.cpp
 -20.9%    -106    ./microbit-v2-samples/libraries/codal-core/source/drivers/TouchButton.cpp
  -4.3%    -106    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitPowerManager.cpp
  -7.5%    -114    ./microbit-v2-samples/libraries/codal-core/source/drivers/MessageBus.cpp
  -9.1%    -114    ./microbit-v2-samples/libraries/codal-core/source/streams/LevelDetectorSPL.cpp
 -42.8%    -118    ./microbit-v2-samples/libraries/codal-core/source/types/Event.cpp
  -6.9%    -126    ./microbit-v2-samples/libraries/codal-core/source/driver-models/Timer.cpp
 -11.1%    -128    ./microbit-v2-samples/libraries/codal-core/source/streams/StreamSplitter.cpp
 -18.8%    -168    ./microbit-v2-samples/libraries/codal-core/source/core/CodalComponent.cpp
  -7.9%    -170    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitUSBFlashManager.cpp
 -11.4%    -180    ./microbit-v2-samples/libraries/codal-microbit-v2/source/Mixer2.cpp
 -16.7%    -182    ./microbit-v2-samples/libraries/codal-core/source/driver-models/Accelerometer.cpp
 -25.0%    -192    ./microbit-v2-samples/libraries/codal-core/source/drivers/Button.cpp
 -35.2%    -200    ./microbit-v2-samples/libraries/codal-core/source/driver-models/I2C.cpp
 -15.9%    -206    ./microbit-v2-samples/libraries/codal-nrf52/source/NRF52I2C.cpp
 -13.3%    -216    ./microbit-v2-samples/libraries/codal-microbit-v2/model/MicroBitIO.cpp
 -12.2%    -220    ./microbit-v2-samples/libraries/codal-core/source/driver-models/Serial.cpp
 -14.1%    -222    ./microbit-v2-samples/libraries/codal-microbit-v2/source/SoundEmojiSynthesizer.cpp
 -16.6%    -228    ./microbit-v2-samples/libraries/codal-core/source/drivers/AnimatedDisplay.cpp
 -20.4%    -228    ./microbit-v2-samples/libraries/codal-core/source/drivers/PulseIn.cpp
 -17.6%    -232    ./microbit-v2-samples/libraries/codal-core/source/types/Image.cpp
 -14.8%    -238    ./microbit-v2-samples/libraries/codal-nrf52/source/NRF52Serial.cpp
 -25.8%    -268    ./microbit-v2-samples/libraries/codal-core/source/drivers/KeyValueStorage.cpp
 -27.2%    -270    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitDevice.cpp
 -21.5%    -270    ./microbit-v2-samples/libraries/codal-microbit-v2/source/SoundOutputPin.cpp
 -25.6%    -414    ./microbit-v2-samples/libraries/codal-nrf52/source/NRF52PWM.cpp
 -36.9%    -532    ./microbit-v2-samples/libraries/codal-nrf52/source/NRFLowLevelTimer.cpp
 -30.3%    -538    ./microbit-v2-samples/libraries/codal-microbit-v2/source/NRF52LedMatrix.cpp
 -23.5%    -598    ./microbit-v2-samples/libraries/codal-microbit-v2/source/MicroBitCompassCalibrator.cpp
 -17.7%    -624    ./microbit-v2-samples/libraries/codal-nrf52/source/NRF52Pin.cpp
 -32.7% -1.10Ki    ./microbit-v2-samples/libraries/codal-nrf52/source/NRF52ADC.cpp
 -34.1% -1.12Ki    ./microbit-v2-samples/libraries/codal-core/source/core/CodalFiber.cpp
 -42.7% -1.46Ki    ./microbit-v2-samples/libraries/codal-microbit-v2/source/SoundExpressions.cpp
  -4.8% -1.46Ki    [section .text]
  [DEL] -2.76Ki    ./microbit-v2-samples/source/samples/AudioTest.cpp
  -3.5% -13.3Ki    TOTAL