diff --git a/setup.py b/setup.py index a1df2f9f5..f2e48bfbd 100644 --- a/setup.py +++ b/setup.py @@ -612,6 +612,7 @@ def my_test_suite(): 'Operating System :: POSIX', 'Programming Language :: C', 'Programming Language :: Python', + 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', diff --git a/src/python-zstd.c b/src/python-zstd.c index 1c3f3357e..807ca5779 100644 --- a/src/python-zstd.c +++ b/src/python-zstd.c @@ -600,6 +600,28 @@ static PyObject *py_zstd_threads_count(PyObject* self, PyObject *args) return Py_BuildValue("i", threads); } +/** + * Returns ZSTD determined cpu count, int + */ +static PyObject *py_zstd_cpu_count_sysconf(PyObject* self, PyObject *args) +{ + UNUSED(self); + UNUSED(args); + int32_t threads = UTIL_countAvailableCores_posix_sysconf(); + return Py_BuildValue("i", threads); +} + +/** + * Returns ZSTD determined cpu count, int + */ +static PyObject *py_zstd_cpu_count_cpuinfo(PyObject* self, PyObject *args) +{ + UNUSED(self); + UNUSED(args); + int32_t threads = UTIL_countAvailableCores_parse_cpuinfo(); + return Py_BuildValue("i", threads); +} + static PyObject *py_zstd_set_cpu_cores_cache_ttl(PyObject* self, PyObject *args) { UNUSED(self); @@ -696,7 +718,9 @@ static PyMethodDef ZstdMethods[] = { {"ZSTD_version_compiled", py_zstd_library_version_compiled, METH_NOARGS, NULL}, {"ZSTD_version_loaded", py_zstd_library_version_loaded, METH_NOARGS, NULL}, {"ZSTD_version_number", py_zstd_library_version_int, METH_NOARGS, ZSTD_INT_VERSION_DOCSTRING}, - {"ZSTD_threads_count", py_zstd_threads_count, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING}, + {"ZSTD_threads_count", py_zstd_cpu_count_sysconf, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING}, + {"ZSTD_cpu_count_sysconf", py_zstd_cpu_count_sysconf, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING}, + {"ZSTD_cpu_count_cpuinfo", py_zstd_cpu_count_cpuinfo, METH_NOARGS, ZSTD_THREADS_COUNT_DOCSTRING}, {"ZSTD_max_threads_count", py_zstd_max_threads_count, METH_NOARGS, ZSTD_MAX_THREADS_COUNT_DOCSTRING}, {"ZSTD_min_compression_level", py_zstd_min_compression_level, METH_NOARGS, ZSTD_MIN_COMPRESSION_LEVEL_DOCSTRING}, {"ZSTD_max_compression_level", py_zstd_max_compression_level, METH_NOARGS, ZSTD_MAX_COMPRESSION_LEVEL_DOCSTRING}, diff --git a/src/util.c b/src/util.c index 5184b0037..2e5ba8732 100644 --- a/src/util.c +++ b/src/util.c @@ -142,10 +142,9 @@ int UTIL_countAvailableCores(void) #elif defined(__linux__) -/* parse /proc/cpuinfo - * siblings / cpu cores should give hyperthreading ratio - * otherwise fall back on sysconf */ -int UTIL_countAvailableCores(void) +/* + * Use only posix sysconf */ +int UTIL_countAvailableCores_posix_sysconf(void) { time_t currTime = time(NULL); if (lastTimeCached && currTime-lastTimeCached>util_cpuCoresCacheTTL) numLogicalCores = 0; @@ -158,11 +157,31 @@ int UTIL_countAvailableCores(void) numLogicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN); if (numLogicalCores == -1) { /* value not queryable, fall back on 1 */ + numLogicalCores = 1; printdn("Sysconf read fail. numLogicalCores: %d\n", numLogicalCores); lastTimeCached = time(NULL); - return numLogicalCores = 1; + return numLogicalCores; } printdn("Sysconf readed. numLogicalCores: %d\n", numLogicalCores); + lastTimeCached = time(NULL); + return numLogicalCores; +} +/* Only parse /proc/cpuinfo + * siblings / cpu cores should give hyperthreading ratio + * otherwise fall back to 1 */ +// Simulate old version +int UTIL_countAvailableCores(void) { + return UTIL_countAvailableCores_posix_sysconf(); +} +int UTIL_countAvailableCores_parse_cpuinfo(void) +{ + time_t currTime = time(NULL); + if (lastTimeCached && currTime-lastTimeCached>util_cpuCoresCacheTTL) numLogicalCores = 0; + + if (numLogicalCores != 0) { + printdn("Stored static numLogicalCores: %d\n", numLogicalCores); + return numLogicalCores; + } /* try to determine if there's hyperthreading */ { FILE* cpuinfo = fopen("/proc/cpuinfo", "r"); diff --git a/src/util.h b/src/util.h index 1aadf169c..c9120110b 100644 --- a/src/util.h +++ b/src/util.h @@ -75,7 +75,13 @@ __attribute__((unused)) static int numLogicalCores = 1; __attribute__((unused)) static time_t lastTimeCached = 0; __attribute__((unused)) static int util_cpuCoresCacheTTL = 60; #endif +#if defined(__linux__) int UTIL_countAvailableCores(void); +int UTIL_countAvailableCores_posix_sysconf(void); +int UTIL_countAvailableCores_prse_cpuinfo(void); +#else +int UTIL_countAvailableCores(void); +#endif int UTIL_setCpuCoresCacheTTL(int cacheTTL); int UTIL_stopCpuCoresCache(void); diff --git a/tests/base.py b/tests/base.py index 681de50a1..8bc883ada 100644 --- a/tests/base.py +++ b/tests/base.py @@ -27,6 +27,8 @@ def raise_skip(msg): log.info("libzstd built with legacy formats support: %r"% zstd.ZSTD_legacy_support()) log.info("zstd max number of threads: %r"% zstd.ZSTD_max_threads_count()) log.info("zstd found CPU cores : %r"% zstd.ZSTD_threads_count()) +log.info("zstd found CPU cores (call sysconf) : %r"% zstd.ZSTD_cpu_count_sysconf()) +log.info("zstd found CPU cores (parse cpuinfo) : %r"% zstd.ZSTD_cpu_count_cpuinfo()) log.info("zstd default compression level: %r"% zstd.ZSTD_default_compression_level()) log.info("zstd max compression level: %r"% zstd.ZSTD_max_compression_level()) log.info("zstd min compression level: %r"% zstd.ZSTD_min_compression_level()) diff --git a/tests/test_speed.py b/tests/test_speed.py index 56d6a48cb..ebc29e48b 100644 --- a/tests/test_speed.py +++ b/tests/test_speed.py @@ -258,7 +258,46 @@ def test_cpu_cores_cache_none_speed(self): endMemoryUsage=get_real_memory_usage() log.info("end Check memory usage = %6.2f kb" % (1.0*endMemoryUsage/1024,)) - log.info("Check cache use speed(0) average = %6.2f Ops/sec" % (1.0*ops/wait,)) + log.info("Check cache use cache(0) and DEFAULT average = %6.2f Ops/sec" % (1.0*ops/wait,)) + log.info("diff Check memory usage = %6.2f kb" % (1.0*(endMemoryUsage-beginMemoryUsage)/1024,)) + + def test_cpu_cores_cache_none_sysconf_speed(self): + wait = 10 + if "ZSTD_FULLTIME_TESTS" in os.environ: + wait = 70 + log.info("\nWait %d seconds..." % wait) + ops = 0 + tbegin = time() + beginMemoryUsage=get_real_memory_usage() + zstd.ZSTD_stopCpuCoresCache() + log.info("begin Check memory usage= %6.2f kb" % (1.0*beginMemoryUsage/1024,)) + while time()-tbegin