Issue 310 use posix sysconf

src/python-zstd.c

@@ -600,6 +600,28 @@ static PyObject *py_zstd_threads_count(PyObject* self, PyObject *args)
     return Py_BuildValue("i", threads);
 }
 
+/**
+ * Returns ZSTD determined cpu count, int
+ */
+static PyObject *py_zstd_cpu_count_sysconf(PyObject* self, PyObject *args)
+{
+    UNUSED(self);
+    UNUSED(args);
+    int32_t threads = UTIL_countAvailableCores_posix_sysconf();
+    return Py_BuildValue("i", threads);
+}
+
+/**
+ * Returns ZSTD determined cpu count, int
+ */
+static PyObject *py_zstd_cpu_count_cpuinfo(PyObject* self, PyObject *args)
+{
+    UNUSED(self);
+    UNUSED(args);
+    int32_t threads = UTIL_countAvailableCores_parse_cpuinfo();
+    return Py_BuildValue("i", threads);
+}
+
 static PyObject *py_zstd_set_cpu_cores_cache_ttl(PyObject* self, PyObject *args)
 {
     UNUSED(self);
@@ -696,7 +718,9 @@ static PyMethodDef ZstdMethods[] = {
     {"ZSTD_version_compiled",  py_zstd_library_version_compiled, METH_NOARGS, NULL},
     {"ZSTD_version_loaded",  py_zstd_library_version_loaded, METH_NOARGS, NULL},
     {"ZSTD_version_number",  py_zstd_library_version_int, METH_NOARGS, ZSTD_INT_VERSION_DOCSTRING},
-    {"ZSTD_threads_count",  py_zstd_threads_count, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING},
+    {"ZSTD_threads_count",  py_zstd_cpu_count_sysconf, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING},
+    {"ZSTD_cpu_count_sysconf",  py_zstd_cpu_count_sysconf, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING},
+    {"ZSTD_cpu_count_cpuinfo",  py_zstd_cpu_count_cpuinfo, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING},
     {"ZSTD_max_threads_count",  py_zstd_max_threads_count, METH_NOARGS, ZSTD_MAX_THREADS_COUNT_DOCSTRING},
     {"ZSTD_min_compression_level",  py_zstd_min_compression_level, METH_NOARGS, ZSTD_MIN_COMPRESSION_LEVEL_DOCSTRING},
     {"ZSTD_max_compression_level",  py_zstd_max_compression_level, METH_NOARGS, ZSTD_MAX_COMPRESSION_LEVEL_DOCSTRING},

src/util.c

@@ -142,10 +142,9 @@ int UTIL_countAvailableCores(void)
 
 #elif defined(__linux__)
 
-/* parse /proc/cpuinfo
- * siblings / cpu cores should give hyperthreading ratio
- * otherwise fall back on sysconf */
-int UTIL_countAvailableCores(void)
+/*
+ * Use only posix sysconf */
+int UTIL_countAvailableCores_posix_sysconf(void)
 {
     time_t currTime = time(NULL);
     if (lastTimeCached && currTime-lastTimeCached>util_cpuCoresCacheTTL) numLogicalCores = 0;
@@ -158,11 +157,31 @@ int UTIL_countAvailableCores(void)
     numLogicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
     if (numLogicalCores == -1) {
         /* value not queryable, fall back on 1 */
+        numLogicalCores = 1;
         printdn("Sysconf read fail. numLogicalCores: %d\n", numLogicalCores);
         lastTimeCached = time(NULL);
-        return numLogicalCores = 1;
+        return numLogicalCores;
     }
 	printdn("Sysconf readed. numLogicalCores: %d\n", numLogicalCores);
+	lastTimeCached = time(NULL);
+	return numLogicalCores;
+}
+/* Only parse /proc/cpuinfo
+ * siblings / cpu cores should give hyperthreading ratio
+ * otherwise fall back to 1 */
+// Simulate old version
+int UTIL_countAvailableCores(void) {
+	return UTIL_countAvailableCores_posix_sysconf();
+}
+int UTIL_countAvailableCores_parse_cpuinfo(void)
+{
+    time_t currTime = time(NULL);
+    if (lastTimeCached && currTime-lastTimeCached>util_cpuCoresCacheTTL) numLogicalCores = 0;
+
+    if (numLogicalCores != 0) {
+        printdn("Stored static numLogicalCores: %d\n", numLogicalCores);
+        return numLogicalCores;
+    }
 
     /* try to determine if there's hyperthreading */
     {   FILE* cpuinfo = fopen("/proc/cpuinfo", "r");

src/util.h

@@ -75,7 +75,13 @@ __attribute__((unused)) static int numLogicalCores = 1;
 __attribute__((unused)) static time_t lastTimeCached = 0;
 __attribute__((unused)) static int util_cpuCoresCacheTTL = 60;
 #endif
+#if defined(__linux__)
 int UTIL_countAvailableCores(void);
+int UTIL_countAvailableCores_posix_sysconf(void);
+int UTIL_countAvailableCores_prse_cpuinfo(void);
+#else
+int UTIL_countAvailableCores(void);
+#endif
 int UTIL_setCpuCoresCacheTTL(int cacheTTL);
 int UTIL_stopCpuCoresCache(void);
 

tests/base.py

@@ -27,6 +27,8 @@ def raise_skip(msg):
 log.info("libzstd built with legacy formats support: %r"% zstd.ZSTD_legacy_support())
 log.info("zstd max number of threads: %r"% zstd.ZSTD_max_threads_count())
 log.info("zstd found CPU cores : %r"% zstd.ZSTD_threads_count())
+log.info("zstd found CPU cores (call sysconf) : %r"% zstd.ZSTD_cpu_count_sysconf())
+log.info("zstd found CPU cores (parse cpuinfo) : %r"% zstd.ZSTD_cpu_count_cpuinfo())
 log.info("zstd default compression level: %r"% zstd.ZSTD_default_compression_level())
 log.info("zstd max compression level: %r"% zstd.ZSTD_max_compression_level())
 log.info("zstd min compression level: %r"% zstd.ZSTD_min_compression_level())

tests/test_speed.py

@@ -258,7 +258,46 @@ def test_cpu_cores_cache_none_speed(self):
 
         endMemoryUsage=get_real_memory_usage()
         log.info("end Check memory usage = %6.2f kb" % (1.0*endMemoryUsage/1024,))
-        log.info("Check cache use speed(0) average = %6.2f Ops/sec" % (1.0*ops/wait,))
+        log.info("Check cache use cache(0) and DEFAULT average = %6.2f Ops/sec" % (1.0*ops/wait,))
+        log.info("diff Check memory usage = %6.2f kb" % (1.0*(endMemoryUsage-beginMemoryUsage)/1024,))
+
+    def test_cpu_cores_cache_none_sysconf_speed(self):
+        wait = 10
+        if "ZSTD_FULLTIME_TESTS" in os.environ:
+            wait = 70
+        log.info("\nWait %d seconds..." % wait)
+        ops = 0
+        tbegin = time()
+        beginMemoryUsage=get_real_memory_usage()
+        zstd.ZSTD_stopCpuCoresCache()
+        log.info("begin Check memory usage= %6.2f kb" % (1.0*beginMemoryUsage/1024,))
+        while time()-tbegin<wait:
+            cores = zstd.ZSTD_cpu_count_sysconf()
+            ops+=1
+
+        endMemoryUsage=get_real_memory_usage()
+        log.info("end Check memory usage = %6.2f kb" % (1.0*endMemoryUsage/1024,))
+        log.info("Check cache use cache(0) and call SYSCONF average = %6.2f Ops/sec" % (1.0*ops/wait,))
+        log.info("diff Check memory usage = %6.2f kb" % (1.0*(endMemoryUsage-beginMemoryUsage)/1024,))
+
+
+    def test_cpu_cores_cache_none_cpuinfo_speed(self):
+        wait = 10
+        if "ZSTD_FULLTIME_TESTS" in os.environ:
+            wait = 70
+        log.info("\nWait %d seconds..." % wait)
+        ops = 0
+        tbegin = time()
+        beginMemoryUsage=get_real_memory_usage()
+        zstd.ZSTD_stopCpuCoresCache()
+        log.info("begin Check memory usage= %6.2f kb" % (1.0*beginMemoryUsage/1024,))
+        while time()-tbegin<wait:
+            cores = zstd.ZSTD_cpu_count_cpuinfo()
+            ops+=1
+
+        endMemoryUsage=get_real_memory_usage()
+        log.info("end Check memory usage = %6.2f kb" % (1.0*endMemoryUsage/1024,))
+        log.info("Check cache use cache(0) and parse CPUINFO average = %6.2f Ops/sec" % (1.0*ops/wait,))
         log.info("diff Check memory usage = %6.2f kb" % (1.0*(endMemoryUsage-beginMemoryUsage)/1024,))
 
     def test_cpu_cores_cache_60_speed(self):