Merge pull request ceph#62165 from athanatos/sjust/wip-recovery-ops-leak-61594-with-history

osd: fix osd mclock queue item leak

Reviewed-by: Sridhar Seshasayee <[email protected]>
Reviewed-by: Radoslaw Zarzynski <[email protected]>

Author: athanatos

.gitmodules

@@ -38,9 +38,6 @@
 [submodule "src/blkin"]
 	path = src/blkin
 	url = https://github.com/ceph/blkin
-[submodule "src/dmclock"]
-	path = src/dmclock
-	url = https://github.com/ceph/dmclock.git
 [submodule "src/seastar"]
 	path = src/seastar
 	url = https://github.com/ceph/seastar.git

src/dmclock

@@ -0,0 +1,5 @@
+*~
+*.dSYM
+*.o
+build*
+cscope.*

src/dmclock/.gitignore

@@ -0,0 +1,27 @@
+language: cpp
+cache: ccache
+dist: focal
+sudo: false
+branches:
+  only:
+    - master
+os:
+  - linux
+compiler:
+  - clang
+  - gcc
+addons:
+  apt:
+    packages:
+      - cmake
+      - libgtest-dev
+      - libboost-dev
+before_script:
+  - mkdir build
+  - cd build
+script:
+  - cmake ..
+  - cmake --build . -- -j2 && ctest -V -j2
+env:
+  global:
+    - LANG="en_US.UTF-8"

src/dmclock/.travis.yml

@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.5.1)
+
+project(dmclock CXX)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules")
+
+if (NOT(BOOST_FOUND))
+  find_package(Boost REQUIRED)
+endif()
+
+find_package(Threads)
+
+if(CMAKE_CXX_STANDARD OR CMAKE_CXX_FLAGS MATCHES "-std=(c|gnu)\\+\\+")
+  # use existing settings if available
+else()
+  set(CMAKE_CXX_STANDARD 11)
+  set(CMAKE_CXX_STANDARD_REQUIRED ON)
+endif()
+
+add_subdirectory(src)
+
+# Determine if dmclock is built as a subproject (using add_subdirectory)
+# or if it is the master project.
+set(MASTER_PROJECT FALSE)
+if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
+  set(MASTER_PROJECT TRUE)
+endif()
+
+option(dmclock_TEST "Generate test targets" ${MASTER_PROJECT})
+if(dmclock_TEST)
+  if (NOT(TARGET gtest AND TARGET gtest_main))
+    if (NOT GTEST_FOUND)
+      find_package(GTest QUIET)
+      if (NOT GTEST_FOUND)
+        include(BuildGTest)
+      endif()
+    endif()
+  endif()
+
+  enable_testing()
+  add_subdirectory(test)
+  add_subdirectory(support/test)
+  add_subdirectory(sim)
+endif()

src/dmclock/CMakeLists.txt

@@ -0,0 +1,45 @@
+# dmclock
+
+This repository contains C++ 11 code that implements the dmclock
+distributed quality of service algorithm. See __mClock: Handling
+Throughput Variability for Hypervisor IO Scheduling__ by Gulati,
+Merchant, and Varman for a description of the algorithm.
+
+## Bugs and features
+
+There is a [dmclock project](https://tracker.ceph.com/projects/dmclock) through
+which bugs can be reported and features requested.
+
+## Running cmake
+
+When running cmake, set the build type with either:
+
+    -DCMAKE_BUILD_TYPE=Debug
+    -DCMAKE_BUILD_TYPE=Release
+
+To turn on profiling, run cmake with an additional:
+
+    -DPROFILE=yes
+
+## Running make
+
+### Building the dmclock library
+
+The `make` command builds a library libdmclock.a. That plus the header
+files in the src directory allow one to use the implementation in
+their code.
+
+### Building unit tests
+
+The `make dmclock-tests` command builds unit tests.
+
+### Building simulations
+
+The `make dmclock-sims` command builds two simulations -- *dmc_sim*
+and *ssched_sim* -- which incorporate, respectively, the dmclock
+priority queue or a very simple scheduler for comparison. Other
+priority queue implementations could be added in the future.
+
+## dmclock API
+
+To be written....

src/dmclock/README.md

@@ -0,0 +1,42 @@
+# dmclock benchmarking
+
+**IMPORTANT**: now that K_WAY_HEAP is no longer allowed to have the
+value 1, the shell and Python scripts that generate the PDFs no longer
+work exactly correctly. Some effort to debug is necessary.
+
+This directory contains scripts to evaluate effects of different
+branching-factors (k=1 to k=11) in the IndirectIntrusiveHeap
+data-structure. IndirectIntrusiveHeap is now a k-way heap, so finding
+an ideal value for k (i.e., k=2 or k=3) for a particular work-load is
+important. Also, it is well-documented that the right choice of
+k-value improves the caching behaviour [Syed -- citation needed
+here]. As a result, the overall performance of an application using
+k-way heap increases significantly [Syed -- citation needed here].
+
+A rule of thumb is the following:
+	if number of elements are <= 6, use k=1
+	otherwise, use k=3.
+
+## Prerequisites
+
+requires python 2.7, gnuplot, and awk.
+  
+## Running benchmark
+
+./run.sh [name_of_the_output] [k_way] [repeat] # [Syed -- last two command line args do not work]
+
+The "run.sh" script looks for config files in the "configs" directory,
+and the final output is generated as
+"name_of_the_output.pdf". Internally, "run.sh" calls other scripts
+such as data_gen.sh, data_parser.py, and plot_gen.sh.
+
+## Modifying parameters
+
+To modify k-value and/or the amount of times each simulation is
+repeated, modify the following two variables in "run.sh" file:
+
+    k_way=[your_value]
+    repeat=[your_value]
+
+For example, k_way=3 means, the benchmark will compare simulations
+using 1-way, 2-way, and 3-way heaps.

src/dmclock/benchmark/README.md

@@ -0,0 +1,31 @@
+[global]
+server_groups = 1
+client_groups = 2
+server_random_selection = true
+server_soft_limit = true
+
+[server.0]
+server_count = 100
+server_iops  = 160
+
+[client.0]
+client_count = 99
+client_wait = 0
+client_total_ops = 10000
+client_server_select_range = 100
+client_iops_goal = 200
+client_outstanding_ops = 32
+client_reservation = 100.0
+client_limit = 0.0
+client_weight = 1.0
+
+[client.1]
+client_count = 1
+client_wait = 10
+client_total_ops = 10000
+client_server_select_range = 100
+client_iops_goal = 200
+client_outstanding_ops = 32
+client_reservation = 100.0
+client_limit = 0.0
+client_weight = 1.0

src/dmclock/benchmark/configs/dmc_sim_100_100.conf

@@ -0,0 +1,43 @@
+[global]
+server_groups = 1
+client_groups = 3
+server_random_selection = true
+server_soft_limit = true
+
+[client.0]
+client_count = 2
+client_wait = 0
+client_total_ops = 1000
+client_server_select_range = 8
+client_iops_goal = 200
+client_outstanding_ops = 32
+client_reservation = 0.0
+client_limit = 0.0
+client_weight = 1.0
+
+[client.1]
+client_count = 2
+client_wait = 5
+client_total_ops = 1000
+client_server_select_range = 8
+client_iops_goal = 200
+client_outstanding_ops = 32
+client_reservation = 20.0
+client_limit = 40.0
+client_weight = 1.0
+
+[client.2]
+client_count = 2
+client_wait = 10
+client_total_ops = 1000
+client_server_select_range = 8
+client_iops_goal = 200
+client_outstanding_ops = 32
+client_reservation = 0.0
+client_limit = 50.0
+client_weight = 2.0
+
+
+[server.0]
+server_count = 8
+server_iops  = 160

src/dmclock/benchmark/configs/dmc_sim_8_6.conf

@@ -0,0 +1,73 @@
+#!/bin/bash
+config_dir="configs"
+repeat=2 #5
+
+# parameter check -- output_file name
+if [ "$1" != "" ]; then
+  output_file="$1"
+else
+  echo "Please provide the name of the output file"
+  exit
+fi
+
+# parameter check -- k-value
+if [ "$2" != "" ]; then
+  k_way="$2"
+else
+  echo "Please provide the maximum K_WAY value"
+  exit
+fi
+
+# parameter check --repeat
+if [ "$3" != "" ]; then
+  repeat="$3"
+fi
+
+echo "k-way:$k_way, num_repeat:$repeat"
+
+# create simulators in different directories 
+k=2
+while [ $k -le $k_way ]
+do
+  mkdir "build_$k"
+  cd "build_$k"
+  rm -rf *
+  cmake -DCMAKE_BUILD_TYPE=Release -DK_WAY_HEAP=$k ../../.
+  make dmclock-sims
+  cd ..
+  
+  k=$(( $k + 1 ))
+done
+
+# run simulators 
+echo '' > $output_file
+for config in "$config_dir"/*.conf
+do
+  k=2
+  while [ $k -le $k_way ]
+  do
+    cd "build_$k"
+    
+    # repeat same experiment
+    i=0
+    while [ $i -lt $repeat ]
+    do  
+      i=$(( $i + 1 ))
+      
+      # clear cache first
+      sync
+      #sudo sh -c 'echo 1 >/proc/sys/vm/drop_caches'
+      #sudo sh -c 'echo 2 >/proc/sys/vm/drop_caches'
+      #sudo sh -c 'echo 3 >/proc/sys/vm/drop_caches'
+
+      # run with heap
+      msg="file_name:$k:$config"
+      echo $msg >> ../$output_file
+      echo "running $msg ..."
+      ./sim/dmc_sim -c ../$config | awk '(/average/)' >> ../$output_file
+    done # end repeat
+    cd ..
+    k=$(( $k + 1 ))
+  done # end k_way
+done # end config
+