diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index eab542d8..85fb8972 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -82,6 +82,7 @@ services: - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN + - AWS_DEFAULT_REGION=$AWS_REGION # for debug # ports: # - "8888:80" diff --git a/src/gprofiler/backend/models/metrics_models.py b/src/gprofiler/backend/models/metrics_models.py index de6945a9..48255e3e 100644 --- a/src/gprofiler/backend/models/metrics_models.py +++ b/src/gprofiler/backend/models/metrics_models.py @@ -70,6 +70,18 @@ class MetricSummary(Metric): class MetricGraph(Metric): uniq_hostnames: Optional[int] time: datetime + avg_frequency: Optional[float] + max_frequency: Optional[float] + avg_cpi_count: Optional[float] + max_cpi_count: Optional[float] + avg_tma_front_end_bound: Optional[float] + max_tma_front_end_bound: Optional[float] + avg_tma_backend_bound: Optional[float] + max_tma_backend_bound: Optional[float] + avg_tma_bad_spec: Optional[float] + max_tma_bad_spec: Optional[float] + avg_tma_retiring: Optional[float] + max_tma_retiring: Optional[float] class MetricNodesCoresSummary(BaseModel): diff --git a/src/gprofiler/frontend/src/api/hooks/useGetServiceMemoryAndCpu.jsx b/src/gprofiler/frontend/src/api/hooks/useGetServiceMetrics.jsx similarity index 86% rename from src/gprofiler/frontend/src/api/hooks/useGetServiceMemoryAndCpu.jsx rename to src/gprofiler/frontend/src/api/hooks/useGetServiceMetrics.jsx index 117aa32c..13b75eb6 100644 --- a/src/gprofiler/frontend/src/api/hooks/useGetServiceMemoryAndCpu.jsx +++ b/src/gprofiler/frontend/src/api/hooks/useGetServiceMetrics.jsx @@ -30,7 +30,7 @@ const areParamsDefined = (selectedService, timeSelection) => { return !_.isUndefined(selectedService) && !_.isUndefined(timeSelection); }; -const useGetServiceMemoryAndCpu = ({ resolution }) => { +const useGetServiceMetrics = ({ resolution }) => { const { selectedService, timeSelection } = useContext(SelectorsContext); const { activeFilterTag } = useContext(FilterTagsContext); const metricsParams = { @@ -41,9 +41,9 @@ const useGetServiceMemoryAndCpu = ({ resolution }) => { const timeParams = getStartEndDateTimeFromSelection(timeSelection); const { - data: memoryAndCpuData, - loading: memoryAndCupLoading, - run: callMemoryAndCpuGraph, + data: metricsData, + loading: metricsLoading, + run: callMetricsGraph, } = useFetchWithRequest( { url: DATA_URLS.GET_GRAPH_METRICS + '?' + stringify(_.assign({ ...timeParams }, metricsParams)), @@ -55,10 +55,10 @@ const useGetServiceMemoryAndCpu = ({ resolution }) => { ); return { - memoryAndCpuData: memoryAndCpuData, - memoryAndCupLoading, - callMemoryAndCpuGraph, + metricsData, + metricsLoading, + callMetricsGraph, }; }; -export default useGetServiceMemoryAndCpu; +export default useGetServiceMetrics; diff --git a/src/gprofiler/frontend/src/components/profiles/views/service/ServiceGraph.jsx b/src/gprofiler/frontend/src/components/profiles/views/service/ServiceGraph.jsx index 73e7b958..97fe91fe 100644 --- a/src/gprofiler/frontend/src/components/profiles/views/service/ServiceGraph.jsx +++ b/src/gprofiler/frontend/src/components/profiles/views/service/ServiceGraph.jsx @@ -19,13 +19,19 @@ import TabContext from '@mui/lab/TabContext'; import { Box } from '@mui/material'; import Typography from '@mui/material/Typography'; -import { memo, useCallback, useContext, useEffect, useState } from 'react'; +import { memo, useCallback, useContext, useState } from 'react'; -import useGetServiceMemoryAndCpu from '@/api/hooks/useGetServiceMemoryAndCpu'; +import useGetServiceMetrics from '@/api/hooks/useGetServiceMetrics'; import useGetServiceNodesAndCores from '@/api/hooks/useGetServiceNodesAndCores'; import useGetServiceSamples from '@/api/hooks/useGetServiceSamples'; import Flexbox from '@/components/common/layout/Flexbox'; +import CpiCountGraph from '@/components/profiles/views/service/graphs/CpiCountGraph'; import ResoultionDropDown from '@/components/profiles/views/service/ResolutionDropDown'; +import TmaBackendBoundGraph from '@/components/profiles/views/service/graphs/TmaBackendBoundGraph'; +import TmaBadSpecGraph from '@/components/profiles/views/service/graphs/TmaBadSpecGraph'; +import TmaFrontEndBoundGraph from '@/components/profiles/views/service/graphs/TmaFrontEndBoundGraph'; +import TmaRetiringGraph from '@/components/profiles/views/service/graphs/TmaRetiringGraph'; +import FrequencyGraph from '@/components/profiles/views/service/graphs/FrequencyGraph'; import { SelectorsContext } from '@/states'; import CoresGraph from './graphs/CoresGraph'; @@ -44,7 +50,7 @@ const ServiceGraph = memo(() => { const { samplesData, samplesLoading } = useGetServiceSamples({ resolution: resolutionValue }); - const { memoryAndCpuData, memoryAndCpuLoading } = useGetServiceMemoryAndCpu({ resolution: resolutionValue }); + const { metricsData, metricsLoading } = useGetServiceMetrics({ resolution: resolutionValue }); const { nodesAndCoresData, nodesAndCoresLoading } = useGetServiceNodesAndCores({ resolution: resolutionValue, }); @@ -65,18 +71,29 @@ const ServiceGraph = memo(() => { const changeTab = (event, newTab) => { setSelectedGraphTab(newTab); }; - + const isTmaFrontEndBoundExist = !!metricsData?.find((metric) => (metric?.avg_tma_front_end_bound || metric?.max_tma_front_end_bound)); + const isTmaBackendBoundExist = !!metricsData?.find((metric) => (metric?.avg_tma_backend_bound || metric?.max_tma_backend_bound)); + const isTmaBadSpecExist = !!metricsData?.find((metric) => (metric?.avg_tma_bad_spec || metric?.max_tma_bad_spec)); + const isTmaRetiringExist = !!metricsData?.find((metric) => (metric?.avg_tma_retiring || metric?.max_tma_retiring)); + const isCpiCountExist = !!metricsData?.find((metric) => (metric?.avg_cpi_count || metric?.max_cpi_count)); + const isFrequencyExist = !!metricsData?.find((metric) => (metric?.avg_frequency || metric?.max_frequency)); return ( - + + {isTmaFrontEndBoundExist && } + {isTmaBackendBoundExist && } + {isTmaBadSpecExist && } + {isTmaRetiringExist && } + {isCpiCountExist && } + {isFrequencyExist && } @@ -89,17 +106,17 @@ const ServiceGraph = memo(() => { - {memoryAndCpuLoading ? ( + {metricsLoading ? ( ) : ( - + )} - {memoryAndCpuLoading ? ( + {metricsLoading ? ( ) : ( - + )} @@ -123,6 +140,48 @@ const ServiceGraph = memo(() => { )} + + {metricsLoading ? ( + + ) : ( + + )} + + + {metricsLoading ? ( + + ) : ( + + )} + + + {metricsLoading ? ( + + ) : ( + + )} + + + {metricsLoading ? ( + + ) : ( + + )} + + + {metricsLoading ? ( + + ) : ( + + )} + + + {metricsLoading ? ( + + ) : ( + + )} + ); diff --git a/src/gprofiler/frontend/src/components/profiles/views/service/graphs/CpiCountGraph.jsx b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/CpiCountGraph.jsx new file mode 100644 index 00000000..4a6ed9e1 --- /dev/null +++ b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/CpiCountGraph.jsx @@ -0,0 +1,42 @@ +{ + /* + * Copyright (C) 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +} + +import 'chartjs-adapter-date-fns'; + +import { CategoryScale, Chart as ChartJS, LinearScale, LineElement, PointElement, TimeScale, Tooltip } from 'chart.js'; +import zoomPlugin from 'chartjs-plugin-zoom'; +import { useEffect, useState } from 'react'; +import { Line } from 'react-chartjs-2'; + +import { calcGraphData, calcGraphOptions } from './graphUtils'; + +ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Tooltip, TimeScale, zoomPlugin); + +const CpiCountGraph = ({ data, edges, setZoomedTime }) => { + const [parsedData, setParsedData] = useState(calcGraphData(data, edges, 'avg_cpi_count', 'max_cpi_count')); + const [options, setOptions] = useState(calcGraphOptions(data, setZoomedTime)); + + useEffect(() => { + setParsedData(calcGraphData(data, edges, 'avg_cpi_count', 'max_cpi_count')); + setOptions(calcGraphOptions(data, setZoomedTime)); + }, [data, edges, setZoomedTime]); + + return ; +}; + +export default CpiCountGraph; diff --git a/src/gprofiler/frontend/src/components/profiles/views/service/graphs/FrequencyGraph.jsx b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/FrequencyGraph.jsx new file mode 100644 index 00000000..c200064e --- /dev/null +++ b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/FrequencyGraph.jsx @@ -0,0 +1,42 @@ +{ + /* + * Copyright (C) 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +} + +import 'chartjs-adapter-date-fns'; + +import { CategoryScale, Chart as ChartJS, LinearScale, LineElement, PointElement, TimeScale, Tooltip } from 'chart.js'; +import zoomPlugin from 'chartjs-plugin-zoom'; +import { useEffect, useState } from 'react'; +import { Line } from 'react-chartjs-2'; + +import { calcGraphData, calcGraphOptions } from './graphUtils'; + +ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Tooltip, TimeScale, zoomPlugin); + +const FrequencyGraph = ({ data, edges, setZoomedTime }) => { + const [parsedData, setParsedData] = useState(calcGraphData(data, edges, 'avg_frequency', 'max_frequency')); + const [options, setOptions] = useState(calcGraphOptions(data, setZoomedTime)); + + useEffect(() => { + setParsedData(calcGraphData(data, edges, 'avg_frequency', 'max_frequency')); + setOptions(calcGraphOptions(data, setZoomedTime)); + }, [data, edges, setZoomedTime]); + + return ; +}; + +export default FrequencyGraph; diff --git a/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaBackendBoundGraph.jsx b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaBackendBoundGraph.jsx new file mode 100644 index 00000000..c9cef1bb --- /dev/null +++ b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaBackendBoundGraph.jsx @@ -0,0 +1,42 @@ +{ + /* + * Copyright (C) 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +} + +import 'chartjs-adapter-date-fns'; + +import { CategoryScale, Chart as ChartJS, LinearScale, LineElement, PointElement, TimeScale, Tooltip } from 'chart.js'; +import zoomPlugin from 'chartjs-plugin-zoom'; +import { useEffect, useState } from 'react'; +import { Line } from 'react-chartjs-2'; + +import { calcGraphData, calcGraphOptions } from './graphUtils'; + +ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Tooltip, TimeScale, zoomPlugin); + +const TmaBackendBoundGraph = ({ data, edges, setZoomedTime }) => { + const [parsedData, setParsedData] = useState(calcGraphData(data, edges, 'avg_tma_backend_bound', 'max_tma_backend_bound')); + const [options, setOptions] = useState(calcGraphOptions(data, setZoomedTime)); + + useEffect(() => { + setParsedData(calcGraphData(data, edges, 'avg_tma_backend_bound', 'max_tma_backend_bound')); + setOptions(calcGraphOptions(data, setZoomedTime)); + }, [data, edges, setZoomedTime]); + + return ; +}; + +export default TmaBackendBoundGraph; diff --git a/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaBadSpecGraph.jsx b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaBadSpecGraph.jsx new file mode 100644 index 00000000..9293d9b5 --- /dev/null +++ b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaBadSpecGraph.jsx @@ -0,0 +1,42 @@ +{ + /* + * Copyright (C) 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +} + +import 'chartjs-adapter-date-fns'; + +import { CategoryScale, Chart as ChartJS, LinearScale, LineElement, PointElement, TimeScale, Tooltip } from 'chart.js'; +import zoomPlugin from 'chartjs-plugin-zoom'; +import { useEffect, useState } from 'react'; +import { Line } from 'react-chartjs-2'; + +import { calcGraphData, calcGraphOptions } from './graphUtils'; + +ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Tooltip, TimeScale, zoomPlugin); + +const TmaBadSpecGraph = ({ data, edges, setZoomedTime }) => { + const [parsedData, setParsedData] = useState(calcGraphData(data, edges, 'avg_tma_bad_spec', 'max_tma_bad_spec')); + const [options, setOptions] = useState(calcGraphOptions(data, setZoomedTime)); + + useEffect(() => { + setParsedData(calcGraphData(data, edges, 'avg_tma_bad_spec', 'max_tma_bad_spec')); + setOptions(calcGraphOptions(data, setZoomedTime)); + }, [data, edges, setZoomedTime]); + + return ; +}; + +export default TmaBadSpecGraph; diff --git a/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaFrontEndBoundGraph.jsx b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaFrontEndBoundGraph.jsx new file mode 100644 index 00000000..311b40cc --- /dev/null +++ b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaFrontEndBoundGraph.jsx @@ -0,0 +1,42 @@ +{ + /* + * Copyright (C) 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +} + +import 'chartjs-adapter-date-fns'; + +import { CategoryScale, Chart as ChartJS, LinearScale, LineElement, PointElement, TimeScale, Tooltip } from 'chart.js'; +import zoomPlugin from 'chartjs-plugin-zoom'; +import { useEffect, useState } from 'react'; +import { Line } from 'react-chartjs-2'; + +import { calcGraphData, calcGraphOptions } from './graphUtils'; + +ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Tooltip, TimeScale, zoomPlugin); + +const TmaFrontEndBoundGraph = ({ data, edges, setZoomedTime }) => { + const [parsedData, setParsedData] = useState(calcGraphData(data, edges, 'avg_tma_front_end_bound', 'max_tma_front_end_bound')); + const [options, setOptions] = useState(calcGraphOptions(data, setZoomedTime)); + + useEffect(() => { + setParsedData(calcGraphData(data, edges, 'avg_tma_front_end_bound', 'max_tma_front_end_bound')); + setOptions(calcGraphOptions(data, setZoomedTime)); + }, [data, edges, setZoomedTime]); + + return ; +}; + +export default TmaFrontEndBoundGraph; diff --git a/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaRetiringGraph.jsx b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaRetiringGraph.jsx new file mode 100644 index 00000000..4f617e52 --- /dev/null +++ b/src/gprofiler/frontend/src/components/profiles/views/service/graphs/TmaRetiringGraph.jsx @@ -0,0 +1,42 @@ +{ + /* + * Copyright (C) 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +} + +import 'chartjs-adapter-date-fns'; + +import { CategoryScale, Chart as ChartJS, LinearScale, LineElement, PointElement, TimeScale, Tooltip } from 'chart.js'; +import zoomPlugin from 'chartjs-plugin-zoom'; +import { useEffect, useState } from 'react'; +import { Line } from 'react-chartjs-2'; + +import { calcGraphData, calcGraphOptions } from './graphUtils'; + +ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Tooltip, TimeScale, zoomPlugin); + +const TmaRetiringGraph = ({ data, edges, setZoomedTime }) => { + const [parsedData, setParsedData] = useState(calcGraphData(data, edges, 'avg_tma_retiring', 'max_tma_retiring')); + const [options, setOptions] = useState(calcGraphOptions(data, setZoomedTime)); + + useEffect(() => { + setParsedData(calcGraphData(data, edges, 'avg_tma_retiring', 'max_tma_retiring')); + setOptions(calcGraphOptions(data, setZoomedTime)); + }, [data, edges, setZoomedTime]); + + return ; +}; + +export default TmaRetiringGraph; diff --git a/src/gprofiler/frontend/src/components/profiles/views/service/serviceGraph.styles.jsx b/src/gprofiler/frontend/src/components/profiles/views/service/serviceGraph.styles.jsx index a1a7d1a9..5980dd99 100644 --- a/src/gprofiler/frontend/src/components/profiles/views/service/serviceGraph.styles.jsx +++ b/src/gprofiler/frontend/src/components/profiles/views/service/serviceGraph.styles.jsx @@ -37,6 +37,7 @@ export const StyledTab = styled(Tab)(({ theme }) => ({ color: theme.palette.grey.dark, minHeight: '40px !important', minWidth: '30px', + fontSize: '14px', padding: '0 10px', '&.Mui-selected': { color: COLORS.BLACK, diff --git a/src/gprofiler_flamedb_rest/Dockerfile b/src/gprofiler_flamedb_rest/Dockerfile index 29035a6e..6e1bf110 100644 --- a/src/gprofiler_flamedb_rest/Dockerfile +++ b/src/gprofiler_flamedb_rest/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.18 as build +FROM golang:1.21 as build ENV GO111MODULE on @@ -14,7 +14,7 @@ RUN go install -v ./... RUN go build -v -o app -FROM debian:10.9-slim +FROM debian:12-slim RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl iputils-ping diff --git a/src/gprofiler_flamedb_rest/common/go.sum b/src/gprofiler_flamedb_rest/common/go.sum index ea950215..7e415bb3 100644 --- a/src/gprofiler_flamedb_rest/common/go.sum +++ b/src/gprofiler_flamedb_rest/common/go.sum @@ -11,6 +11,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= @@ -66,14 +68,19 @@ golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= diff --git a/src/gprofiler_flamedb_rest/common/params.go b/src/gprofiler_flamedb_rest/common/params.go index d3b16e4a..2163de38 100644 --- a/src/gprofiler_flamedb_rest/common/params.go +++ b/src/gprofiler_flamedb_rest/common/params.go @@ -133,14 +133,26 @@ type SamplesCountByFunction struct { } type MetricsSummary struct { - AvgCpu float64 `json:"avg_cpu"` - MaxCpu float64 `json:"max_cpu"` - AvgMemory float64 `json:"avg_memory"` - PercentileMemory float64 `json:"percentile_memory"` - MaxMemory float64 `json:"max_memory"` - UniqHostnames int `json:"uniq_hostnames,omitempty"` - GroupedBy *string `json:"grouped_by,omitempty"` - Time *time.Time `json:"time,omitempty"` + AvgCpu float64 `json:"avg_cpu"` + MaxCpu float64 `json:"max_cpu"` + AvgMemory float64 `json:"avg_memory"` + MaxMemory float64 `json:"max_memory"` + PercentileMemory float64 `json:"percentile_memory"` + AvgFrequency *float64 `json:"avg_frequency,omitempty"` + MaxFrequency *float64 `json:"max_frequency,omitempty"` + AvgCPICount *float64 `json:"avg_cpi_count,omitempty"` + MaxCPICount *float64 `json:"max_cpi_count,omitempty"` + AvgTMAFrontEndBound *float64 `json:"avg_tma_front_end_bound,omitempty"` + MaxTMAFrontEndBound *float64 `json:"max_tma_front_end_bound,omitempty"` + AvgTMABackendBound *float64 `json:"avg_tma_backend_bound,omitempty"` + MaxTMABackendBound *float64 `json:"max_tma_backend_bound,omitempty"` + AvgTMABadSpec *float64 `json:"avg_tma_bad_spec,omitempty"` + MaxTMABadSpec *float64 `json:"max_tma_bad_spec,omitempty"` + AvgTMARetiring *float64 `json:"avg_tma_retiring,omitempty"` + MaxTMARetiring *float64 `json:"max_tma_retiring,omitempty"` + UniqHostnames int `json:"uniq_hostnames,omitempty"` + GroupedBy *string `json:"grouped_by,omitempty"` + Time *time.Time `json:"time,omitempty"` } type MetricsCpuTrend struct { diff --git a/src/gprofiler_flamedb_rest/db/clickhouse.go b/src/gprofiler_flamedb_rest/db/clickhouse.go index ddbe038f..db37d681 100644 --- a/src/gprofiler_flamedb_rest/db/clickhouse.go +++ b/src/gprofiler_flamedb_rest/db/clickhouse.go @@ -21,7 +21,6 @@ import ( "database/sql" "errors" "fmt" - "github.com/ClickHouse/clickhouse-go" "log" "math" "regexp" @@ -31,6 +30,8 @@ import ( "strings" "sync" "time" + + "github.com/ClickHouse/clickhouse-go" ) const ( @@ -268,6 +269,7 @@ func BuildConditions(ContainerName []string, HostName []string, InstanceType []s } func NewClickHouseClient(addr string) *ClickHouseClient { + fmt.Printf("clickhouse addres: %s\n\n", addr) db, err := sql.Open("clickhouse", "tcp://"+addr) if err != nil { log.Fatal(err) @@ -543,7 +545,7 @@ func (c *ClickHouseClient) FetchSampleCountByFunction(ctx context.Context, param GROUP BY Datetime ORDER BY Datetime DESC ) - + SELECT (function_samples.sum_cpu/all_samples.sum_cpu) AS Samples , all_samples.Datetime AS Datetime FROM all_samples LEFT JOIN function_samples ON function_samples.Datetime = all_samples.Datetime; @@ -806,19 +808,35 @@ func (c *ClickHouseClient) FetchMetricsGraph(ctx context.Context, params common. groupBy = fmt.Sprintf(", %s", params.GroupBy) } query := fmt.Sprintf(` - SELECT Datetime %s, arrayAvg(flatten(groupArray(CPUArray))), MAX(MaxCPU), - AVG(MaxMemory), MAX(MaxMemory), quantile(%f)(MaxMemory) FROM - (SELECT toStartOfInterval(Timestamp, INTERVAL '%s') as - Datetime %s, - HostName, - MAX(MemoryAverageUsedPercent) AS MaxMemory, - MAX(CPUAverageUsedPercent) as MaxCPU, - groupArray(CPUAverageUsedPercent) as CPUArray - FROM %s - WHERE ServiceId = %d AND (Datetime BETWEEN '%s' AND '%s') %s - GROUP BY Datetime %s, HostName) GROUP BY Datetime %s ORDER BY Datetime DESC; + SELECT Datetime %s, + arrayAvg(flatten(groupArray(CPUArray))), MAX(MaxCPU), + AVG(MaxMemory), MAX(MaxMemory), quantile(%f)(MaxMemory), + AVG(MaxFrequency), MAX(MaxFrequency), + AVG(MaxCPICount), MAX(MaxCPICount), + AVG(MaxTMAFrontEndBound), MAX(MaxTMAFrontEndBound), + AVG(MaxTMABackendBound), MAX(MaxTMABackendBound), + AVG(MaxTMABadSpec), MAX(MaxTMABadSpec), + AVG(MaxTMARetiring), MAX(MaxTMARetiring) + FROM ( + SELECT toStartOfInterval(Timestamp, INTERVAL '%s') as Datetime %s, + HostName, + MAX(MemoryAverageUsedPercent) AS MaxMemory, + MAX(CPUAverageUsedPercent) as MaxCPU, + groupArray(CPUAverageUsedPercent) as CPUArray, + MAX(CPUFrequency) AS MaxFrequency, + MAX(CPUCPI) AS MaxCPICount, + MAX(CPUTMAFrontEndBound) AS MaxTMAFrontEndBound, + MAX(CPUTMABackendBound) AS MaxTMABackendBound, + MAX(CPUTMABadSpec) AS MaxTMABadSpec, + MAX(CPUTMARetiring) AS MaxTMARetiring + FROM %s + WHERE ServiceId = %d AND (Datetime BETWEEN '%s' AND '%s') %s + GROUP BY Datetime %s, HostName + ) + GROUP BY Datetime %s ORDER BY Datetime DESC; `, groupBy, percentile, interval, groupBy, config.ClickHouseMetricsTable, params.ServiceId, common.FormatTime(params.StartDateTime), common.FormatTime(params.EndDateTime), conditions, groupBy, groupBy) + rows, err := c.client.Query(query) if err == nil { defer func(rows *sql.Rows) { @@ -830,29 +848,63 @@ func (c *ClickHouseClient) FetchMetricsGraph(ctx context.Context, params common. for rows.Next() { var timestamp time.Time var groupedBy string - var avgCpu float64 - var maxCpu float64 - var avgMemory float64 - var maxMemory float64 - var percentileMemory float64 + var avgCpu, maxCpu float64 + var avgMemory, maxMemory, percentileMemory float64 var groupedByPtr *string + + var avgFrequency, maxFrequency sql.NullFloat64 + var avgCPICount, maxCPICount sql.NullFloat64 + var avgTMAFrontEndBound, maxTMAFrontEndBound sql.NullFloat64 + var avgTMABackendBound, maxTMABackendBound sql.NullFloat64 + var avgTMABadSpec, maxTMABadSpec sql.NullFloat64 + var avgTMARetiring, maxTMARetiring sql.NullFloat64 + if params.GroupBy != "none" { - err = rows.Scan(×tamp, &groupedBy, &avgCpu, &maxCpu, &avgMemory, &maxMemory, &percentileMemory) + err = rows.Scan( + ×tamp, &groupedBy, &avgCpu, &maxCpu, &avgMemory, &maxMemory, &percentileMemory, + &avgFrequency, &maxFrequency, + &avgCPICount, &maxCPICount, + &avgTMAFrontEndBound, &maxTMAFrontEndBound, + &avgTMABackendBound, &maxTMABackendBound, + &avgTMABadSpec, &maxTMABadSpec, + &avgTMARetiring, &maxTMARetiring, + ) groupedByPtr = &groupedBy } else { - err = rows.Scan(×tamp, &avgCpu, &maxCpu, &avgMemory, &maxMemory, &percentileMemory) + err = rows.Scan( + ×tamp, &avgCpu, &maxCpu, &avgMemory, &maxMemory, &percentileMemory, + &avgFrequency, &maxFrequency, + &avgCPICount, &maxCPICount, + &avgTMAFrontEndBound, &maxTMAFrontEndBound, + &avgTMABackendBound, &maxTMABackendBound, + &avgTMABadSpec, &maxTMABadSpec, + &avgTMARetiring, &maxTMARetiring, + ) } if err != nil { log.Printf("error scan result: %v", err) } + result = append(result, common.MetricsSummary{ - AvgCpu: avgCpu, - MaxCpu: maxCpu, - AvgMemory: avgMemory, - MaxMemory: maxMemory, - PercentileMemory: percentileMemory, - GroupedBy: groupedByPtr, - Time: ×tamp, + AvgCpu: avgCpu, + MaxCpu: maxCpu, + AvgMemory: avgMemory, + MaxMemory: maxMemory, + PercentileMemory: percentileMemory, + AvgFrequency: getNullableFloat(avgFrequency), + MaxFrequency: getNullableFloat(maxFrequency), + AvgCPICount: getNullableFloat(avgCPICount), + MaxCPICount: getNullableFloat(maxCPICount), + AvgTMAFrontEndBound: getNullableFloat(avgTMAFrontEndBound), + MaxTMAFrontEndBound: getNullableFloat(maxTMAFrontEndBound), + AvgTMABackendBound: getNullableFloat(avgTMABackendBound), + MaxTMABackendBound: getNullableFloat(maxTMABackendBound), + AvgTMABadSpec: getNullableFloat(avgTMABadSpec), + MaxTMABadSpec: getNullableFloat(maxTMABadSpec), + AvgTMARetiring: getNullableFloat(avgTMARetiring), + MaxTMARetiring: getNullableFloat(maxTMARetiring), + GroupedBy: groupedByPtr, + Time: ×tamp, }) } err = rows.Err() @@ -862,6 +914,13 @@ func (c *ClickHouseClient) FetchMetricsGraph(ctx context.Context, params common. return result, err } +func getNullableFloat(val sql.NullFloat64) *float64 { + if val.Valid { + return &val.Float64 + } + return nil +} + func (c *ClickHouseClient) FetchMetricsCpuTrend(ctx context.Context, params common.MetricsCpuTrendParams, filterQuery string) (common.MetricsCpuTrend, error) { defaultEmptyList := make([]string, 0) @@ -870,11 +929,11 @@ func (c *ClickHouseClient) FetchMetricsCpuTrend(ctx context.Context, params comm query := fmt.Sprintf(` WITH CURRENT_CONSUMPTION AS ( - SELECT - arrayAvg(flatten(groupArray(CPUArray))) AS avg_cpu, + SELECT + arrayAvg(flatten(groupArray(CPUArray))) AS avg_cpu, MAX(MaxCPU) AS max_cpu, - AVG(MaxMemory) AS avg_memory, - MAX(MaxMemory) AS max_memory, + AVG(MaxMemory) AS avg_memory, + MAX(MaxMemory) AS max_memory, 1 SortOrder FROM (SELECT @@ -886,11 +945,11 @@ func (c *ClickHouseClient) FetchMetricsCpuTrend(ctx context.Context, params comm GROUP BY HostName) ), PREVIOUS_CONSUMPTION AS ( - SELECT - arrayAvg(flatten(groupArray(CPUArray))) AS avg_cpu, + SELECT + arrayAvg(flatten(groupArray(CPUArray))) AS avg_cpu, MAX(MaxCPU) AS max_cpu, - AVG(MaxMemory) AS avg_memory, - MAX(MaxMemory) AS max_memory, + AVG(MaxMemory) AS avg_memory, + MAX(MaxMemory) AS max_memory, 2 SortOrder FROM (SELECT @@ -901,7 +960,7 @@ func (c *ClickHouseClient) FetchMetricsCpuTrend(ctx context.Context, params comm WHERE ServiceId = %d AND (Timestamp BETWEEN '%s' AND '%s') %s GROUP BY HostName) ) - SELECT avg_cpu, max_cpu, avg_memory, max_memory + SELECT avg_cpu, max_cpu, avg_memory, max_memory FROM ( SELECT * FROM CURRENT_CONSUMPTION UNION ALL diff --git a/src/gprofiler_flamedb_rest/go.mod b/src/gprofiler_flamedb_rest/go.mod index da360767..370a06c2 100644 --- a/src/gprofiler_flamedb_rest/go.mod +++ b/src/gprofiler_flamedb_rest/go.mod @@ -1,6 +1,8 @@ module restflamedb -go 1.18 +go 1.21.0 + +toolchain go1.24.0 replace restflamedb/db => ./db @@ -11,9 +13,9 @@ replace restflamedb/handlers => ./handlers replace restflamedb/config => ./config require ( - github.com/gin-contrib/cors v1.5.0 + github.com/gin-contrib/cors v1.7.3 github.com/gin-contrib/gzip v0.0.6 - github.com/gin-gonic/gin v1.9.1 + github.com/gin-gonic/gin v1.10.0 restflamedb/common v0.0.0-00010101000000-000000000000 restflamedb/config v0.0.0-00010101000000-000000000000 restflamedb/db v0.0.0-00010101000000-000000000000 @@ -24,33 +26,36 @@ require ( github.com/ClickHouse/clickhouse-go v1.5.4 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/a8m/rql v1.3.0 // indirect - github.com/bytedance/sonic v1.10.2 // indirect + github.com/bytedance/sonic v1.12.6 // indirect + github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect - github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.7 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.17.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect + github.com/go-playground/validator/v10 v10.23.0 // indirect + github.com/goccy/go-json v0.10.4 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect - github.com/leodido/go-urn v1.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.7.1 // indirect - github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - golang.org/x/arch v0.6.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.19.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/arch v0.12.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + google.golang.org/protobuf v1.36.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/src/gprofiler_flamedb_rest/go.sum b/src/gprofiler_flamedb_rest/go.sum index 6d257fc7..7021ddd2 100644 --- a/src/gprofiler_flamedb_rest/go.sum +++ b/src/gprofiler_flamedb_rest/go.sum @@ -12,6 +12,11 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= +github.com/bytedance/sonic v1.12.6 h1:/isNmCUF2x3Sh8RAp/4mh4ZGkcFAX/hLrzrK3AvpRzk= +github.com/bytedance/sonic v1.12.6/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= +github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= @@ -21,6 +26,10 @@ github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0 github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -29,8 +38,12 @@ github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xb github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= +github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= github.com/gin-contrib/cors v1.5.0 h1:DgGKV7DDoOn36DFkNtbHrjoRiT5ExCe+PC9/xp7aKvk= github.com/gin-contrib/cors v1.5.0/go.mod h1:TvU7MAZ3EwrPLI2ztzTt3tqgvBCq+wn8WpZmfADjupI= +github.com/gin-contrib/cors v1.7.3 h1:hV+a5xp8hwJoTw7OY+a70FsL8JkVVFTXw9EcfrYUdns= +github.com/gin-contrib/cors v1.7.3/go.mod h1:M3bcKZhxzsvI+rlRSkkxHyljJt1ESd93COUvemZ79j4= github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -38,6 +51,8 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -52,11 +67,15 @@ github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqR github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-playground/validator/v10 v10.17.0 h1:SmVVlfAOtlZncTxRuinDPomC2DkXJ4E5T9gDA0AIH74= github.com/go-playground/validator/v10 v10.17.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= +github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= +github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= @@ -73,6 +92,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -85,6 +106,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -104,6 +127,8 @@ github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -133,6 +158,8 @@ github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc= golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= +golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -141,6 +168,8 @@ golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -148,6 +177,8 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -159,12 +190,16 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= @@ -172,6 +207,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/src/gprofiler_indexer/callstacks.go b/src/gprofiler_indexer/callstacks.go index 2dee5430..fa8eec92 100644 --- a/src/gprofiler_indexer/callstacks.go +++ b/src/gprofiler_indexer/callstacks.go @@ -49,8 +49,14 @@ type FileInfo struct { } `json:"run_arguments"` } `json:"metadata"` Metrics struct { - CPUAvg float64 `json:"cpu_avg"` - MemoryAvg float64 `json:"mem_avg"` + CPUAvg float64 `json:"cpu_avg"` + MemoryAvg float64 `json:"mem_avg"` + CPUFrequency *float64 `json:"cpu_freq,omitempty"` + CPUCPI *float64 `json:"cpu_cpi,omitempty"` + CPUTMAFEBound *float64 `json:"cpu_tma_fe_bound,omitempty"` + CPUTMABEBound *float64 `json:"cpu_tma_be_bound,omitempty"` + CPUTMABadSpec *float64 `json:"cpu_tma_bad_spec,omitempty"` + CPUTMARetiring *float64 `json:"cpu_tma_retiring,omitempty"` } `json:"metrics"` ApplicationMetadataEnabled bool `json:"application_metadata_enabled"` } @@ -204,7 +210,14 @@ func (pw *ProfilesWriter) writeStacks(weights FrameValuesMap, frames map[string] } func (pw *ProfilesWriter) writeMetrics(serviceId uint32, instanceType string, - hostname string, timestamp time.Time, cpuAverageUsedPercent float64, memoryAverageUsedPercent float64) { + hostname string, timestamp time.Time, cpuAverageUsedPercent float64, memoryAverageUsedPercent float64, + cpuFrequency *float64, + cpuCPI *float64, + cpuTMAFrontEndBound *float64, + cpuTMABackendBound *float64, + cpuTMABadSpec *float64, + cpuTMARetiring *float64, +) { metricRecord := MetricRecord{ Timestamp: timestamp, @@ -213,6 +226,12 @@ func (pw *ProfilesWriter) writeMetrics(serviceId uint32, instanceType string, HostName: hostname, CPUAverageUsedPercent: cpuAverageUsedPercent, MemoryAverageUsedPercent: memoryAverageUsedPercent, + CPUFrequency: cpuFrequency, + CPUCPI: cpuCPI, + CPUTMAFrontEndBound: cpuTMAFrontEndBound, + CPUTMABackendBound: cpuTMABackendBound, + CPUTMABadSpec: cpuTMABadSpec, + CPUTMARetiring: cpuTMARetiring, } pw.metricsRecords <- metricRecord } @@ -267,7 +286,9 @@ func (pw *ProfilesWriter) ParseStackFrameFile(serviceId int, timestamp time.Time if fileInfo.Metrics.CPUAvg != 0 && fileInfo.Metrics.MemoryAvg != 0 { pw.writeMetrics(uint32(serviceId), fileInfo.Metadata.CloudInfo.InstanceType, - fileInfo.Metadata.Hostname, timestamp, fileInfo.Metrics.CPUAvg, fileInfo.Metrics.MemoryAvg) + fileInfo.Metadata.Hostname, timestamp, fileInfo.Metrics.CPUAvg, fileInfo.Metrics.MemoryAvg, + fileInfo.Metrics.CPUFrequency, fileInfo.Metrics.CPUCPI, fileInfo.Metrics.CPUTMAFEBound, + fileInfo.Metrics.CPUTMABEBound, fileInfo.Metrics.CPUTMABadSpec, fileInfo.Metrics.CPUTMARetiring) } return nil diff --git a/src/gprofiler_indexer/clickhouse.go b/src/gprofiler_indexer/clickhouse.go index 995aacb7..02ff36d9 100644 --- a/src/gprofiler_indexer/clickhouse.go +++ b/src/gprofiler_indexer/clickhouse.go @@ -20,9 +20,10 @@ import ( "context" "crypto/tls" "fmt" - "github.com/ClickHouse/clickhouse-go/v2" "sync" "time" + + "github.com/ClickHouse/clickhouse-go/v2" ) type StackRecord struct { @@ -47,6 +48,12 @@ type MetricRecord struct { HostName string CPUAverageUsedPercent float64 MemoryAverageUsedPercent float64 + CPUFrequency *float64 + CPUCPI *float64 + CPUTMAFrontEndBound *float64 + CPUTMABackendBound *float64 + CPUTMABadSpec *float64 + CPUTMARetiring *float64 } type RecordsAttributesUnpack interface { @@ -61,6 +68,12 @@ func (mr MetricRecord) getDbAttributes() []interface{} { mr.HostName, mr.CPUAverageUsedPercent, mr.MemoryAverageUsedPercent, + mr.CPUFrequency, + mr.CPUCPI, + mr.CPUTMAFrontEndBound, + mr.CPUTMABackendBound, + mr.CPUTMABadSpec, + mr.CPUTMARetiring, } return dbAttributes } diff --git a/src/gprofiler_indexer/sql/create_ch_schema.sql b/src/gprofiler_indexer/sql/create_ch_schema.sql index a3516d81..635e169c 100644 --- a/src/gprofiler_indexer/sql/create_ch_schema.sql +++ b/src/gprofiler_indexer/sql/create_ch_schema.sql @@ -13,235 +13,303 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- - -- create database on cluster -CREATE DATABASE IF NOT EXISTS - flamedb; - +CREATE DATABASE IF NOT EXISTS flamedb; -- create raw table samples local -CREATE TABLE IF NOT EXISTS flamedb.samples -( - Timestamp DateTime('UTC') CODEC (DoubleDelta), - ServiceId UInt32, - InstanceType LowCardinality(String), - ContainerEnvName LowCardinality(String), - HostName LowCardinality(String), - ContainerName LowCardinality(String), - NumSamples UInt32 CODEC (DoubleDelta), - CallStackHash UInt64, - HostNameHash UInt32 MATERIALIZED xxHash32(HostName), - ContainerNameHash UInt32 MATERIALIZED xxHash32(ContainerName), - CallStackName String CODEC (ZSTD), - CallStackParent UInt64, - InsertionTimestamp DateTime('UTC') CODEC (DoubleDelta), - ErrNumSamples UInt32 -) engine = MergeTree() PARTITION BY toYYYYMMDD(Timestamp) - ORDER BY (ServiceId, InstanceType, ContainerEnvName, HostNameHash, ContainerNameHash, Timestamp); +CREATE TABLE IF NOT EXISTS flamedb.samples ( + Timestamp DateTime ('UTC') CODEC (DoubleDelta), + ServiceId UInt32, + InstanceType LowCardinality (String), + ContainerEnvName LowCardinality (String), + HostName LowCardinality (String), + ContainerName LowCardinality (String), + NumSamples UInt32 CODEC (DoubleDelta), + CallStackHash UInt64, + HostNameHash UInt32 MATERIALIZED xxHash32 (HostName), + ContainerNameHash UInt32 MATERIALIZED xxHash32 (ContainerName), + CallStackName String CODEC (ZSTD), + CallStackParent UInt64, + InsertionTimestamp DateTime ('UTC') CODEC (DoubleDelta), + ErrNumSamples UInt32 +) engine = MergeTree () +PARTITION BY + toYYYYMMDD (Timestamp) +ORDER BY + ( + ServiceId, + InstanceType, + ContainerEnvName, + HostNameHash, + ContainerNameHash, + Timestamp + ); -- create raw table metrics local -CREATE TABLE IF NOT EXISTS flamedb.metrics -( - Timestamp DateTime('UTC') CODEC (DoubleDelta), - ServiceId UInt32, - InstanceType LowCardinality(String), - HostName LowCardinality(String), - HostNameHash UInt32 MATERIALIZED xxHash32(HostName), - CPUAverageUsedPercent Float64, - MemoryAverageUsedPercent Float64 -) engine = MergeTree() PARTITION BY toYYYYMMDD(Timestamp) - ORDER BY (ServiceId, InstanceType, HostNameHash, Timestamp); - +CREATE TABLE IF NOT EXISTS flamedb.metrics ( + Timestamp DateTime ('UTC') CODEC (DoubleDelta), + ServiceId UInt32, + InstanceType LowCardinality (String), + HostName LowCardinality (String), + HostNameHash UInt32 MATERIALIZED xxHash32 (HostName), + CPUAverageUsedPercent Float64, + MemoryAverageUsedPercent Float64, + CPUFrequency Nullable (Float64), + CPUCPI Nullable (Float64), + CPUTMAFrontEndBound Nullable (Float64), + CPUTMABackendBound Nullable (Float64), + CPUTMABadSpec Nullable (Float64), + CPUTMARetiring Nullable (Float64) +) engine = MergeTree () +PARTITION BY + toYYYYMMDD (Timestamp) +ORDER BY + (ServiceId, InstanceType, HostNameHash, Timestamp); -- create 60min aggregated table all hostnames and all containers -CREATE TABLE IF NOT EXISTS flamedb.samples_1hour_all -( - Timestamp DateTime('UTC') CODEC(DoubleDelta), +CREATE TABLE IF NOT EXISTS flamedb.samples_1hour_all ( + Timestamp DateTime ('UTC') CODEC (DoubleDelta), ServiceId UInt32, - CallStackHash UInt64, - CallStackName String CODEC (ZSTD), - CallStackParent UInt64, - InsertionTimestamp DateTime('UTC') CODEC(DoubleDelta), - NumSamples UInt64 CODEC(DoubleDelta), - ErrNumSamples UInt32 -) ENGINE = SummingMergeTree((NumSamples)) - PARTITION BY toYYYYMMDD(Timestamp) - ORDER BY (ServiceId, Timestamp, CallStackHash, CallStackParent); - -CREATE MATERIALIZED VIEW IF NOT EXISTS - flamedb.samples_1hour_all_mv TO flamedb.samples_1hour_all -AS -SELECT toStartOfHour(Timestamp) AS Timestamp, - ServiceId, - CallStackHash, - any(CallStackName) as CallStackName, - any(CallStackParent) as CallStackParent, - sum(NumSamples) AS NumSamples, - sum(ErrNumSamples) AS ErrNumSamples, - anyLast(InsertionTimestamp) as InsertionTimestamp -FROM flamedb.samples -GROUP BY ServiceId, CallStackHash, Timestamp; - + CallStackHash UInt64, + CallStackName String CODEC (ZSTD), + CallStackParent UInt64, + InsertionTimestamp DateTime ('UTC') CODEC (DoubleDelta), + NumSamples UInt64 CODEC (DoubleDelta), + ErrNumSamples UInt32 +) ENGINE = SummingMergeTree ((NumSamples)) +PARTITION BY + toYYYYMMDD (Timestamp) +ORDER BY + ( + ServiceId, + Timestamp, + CallStackHash, + CallStackParent + ); +CREATE MATERIALIZED VIEW IF NOT EXISTS flamedb.samples_1hour_all_mv TO flamedb.samples_1hour_all AS +SELECT + toStartOfHour (Timestamp) AS Timestamp, + ServiceId, + CallStackHash, + any (CallStackName) as CallStackName, + any (CallStackParent) as CallStackParent, + sum(NumSamples) AS NumSamples, + sum(ErrNumSamples) AS ErrNumSamples, + anyLast (InsertionTimestamp) as InsertionTimestamp +FROM + flamedb.samples +GROUP BY + ServiceId, + CallStackHash, + Timestamp; -- create 60min aggregated table -CREATE TABLE IF NOT EXISTS flamedb.samples_1hour -( - Timestamp DateTime('UTC') CODEC(DoubleDelta), +CREATE TABLE IF NOT EXISTS flamedb.samples_1hour ( + Timestamp DateTime ('UTC') CODEC (DoubleDelta), ServiceId UInt32, - InstanceType LowCardinality(String), - ContainerEnvName LowCardinality(String), - HostName LowCardinality(String), - ContainerName LowCardinality(String), - CallStackHash UInt64, - CallStackName String CODEC (ZSTD), - CallStackParent UInt64, - InsertionTimestamp DateTime('UTC') CODEC(DoubleDelta), - NumSamples UInt64 CODEC(DoubleDelta), - ErrNumSamples UInt32, -) ENGINE = SummingMergeTree((NumSamples)) - PARTITION BY toYYYYMMDD(Timestamp) - ORDER BY (ServiceId, ContainerEnvName, InstanceType, HostName, ContainerName, - Timestamp, CallStackHash, CallStackParent); - -CREATE MATERIALIZED VIEW IF NOT EXISTS - flamedb.samples_1hour_mv TO flamedb.samples_1hour -AS -SELECT toStartOfHour(Timestamp) AS Timestamp, - ServiceId, - ContainerEnvName, - InstanceType, - HostName, - ContainerName, - CallStackHash, - any(CallStackName) as CallStackName, - any(CallStackParent) as CallStackParent, - sum(NumSamples) AS NumSamples, - sum(ErrNumSamples) AS ErrNumSamples, - anyLast(InsertionTimestamp) as InsertionTimestamp -FROM flamedb.samples -GROUP BY ServiceId, InstanceType, ContainerEnvName, HostName, ContainerName, - CallStackHash, Timestamp; + InstanceType LowCardinality (String), + ContainerEnvName LowCardinality (String), + HostName LowCardinality (String), + ContainerName LowCardinality (String), + CallStackHash UInt64, + CallStackName String CODEC (ZSTD), + CallStackParent UInt64, + InsertionTimestamp DateTime ('UTC') CODEC (DoubleDelta), + NumSamples UInt64 CODEC (DoubleDelta), + ErrNumSamples UInt32, +) ENGINE = SummingMergeTree ((NumSamples)) +PARTITION BY + toYYYYMMDD (Timestamp) +ORDER BY + ( + ServiceId, + ContainerEnvName, + InstanceType, + HostName, + ContainerName, + Timestamp, + CallStackHash, + CallStackParent + ); +CREATE MATERIALIZED VIEW IF NOT EXISTS flamedb.samples_1hour_mv TO flamedb.samples_1hour AS +SELECT + toStartOfHour (Timestamp) AS Timestamp, + ServiceId, + ContainerEnvName, + InstanceType, + HostName, + ContainerName, + CallStackHash, + any (CallStackName) as CallStackName, + any (CallStackParent) as CallStackParent, + sum(NumSamples) AS NumSamples, + sum(ErrNumSamples) AS ErrNumSamples, + anyLast (InsertionTimestamp) as InsertionTimestamp +FROM + flamedb.samples +GROUP BY + ServiceId, + InstanceType, + ContainerEnvName, + HostName, + ContainerName, + CallStackHash, + Timestamp; -- create distributed 24h table all hostnames and all containers -CREATE TABLE IF NOT EXISTS flamedb.samples_1day_all -( - Timestamp DateTime('UTC') CODEC(DoubleDelta), +CREATE TABLE IF NOT EXISTS flamedb.samples_1day_all ( + Timestamp DateTime ('UTC') CODEC (DoubleDelta), ServiceId UInt32, - CallStackHash UInt64, - CallStackName String CODEC (ZSTD), - CallStackParent UInt64, - InsertionTimestamp DateTime('UTC') CODEC(DoubleDelta), - NumSamples UInt64 CODEC(DoubleDelta), - ErrNumSamples UInt32 -) ENGINE = SummingMergeTree((NumSamples)) - PARTITION BY toYYYYMMDD(Timestamp) - ORDER BY (ServiceId, Timestamp, CallStackHash, CallStackParent); - - -CREATE MATERIALIZED VIEW IF NOT EXISTS - flamedb.samples_1day_all_mv to flamedb.samples_1day_all -AS -SELECT toStartOfDay(Timestamp) AS Timestamp, - ServiceId, - CallStackHash, - any(CallStackName) as CallStackName, - any(CallStackParent) as CallStackParent, - sum(NumSamples) AS NumSamples, - sum(ErrNumSamples) AS ErrNumSamples, - anyLast(InsertionTimestamp) as InsertionTimestamp -FROM flamedb.samples -GROUP BY ServiceId, CallStackHash, Timestamp; + CallStackHash UInt64, + CallStackName String CODEC (ZSTD), + CallStackParent UInt64, + InsertionTimestamp DateTime ('UTC') CODEC (DoubleDelta), + NumSamples UInt64 CODEC (DoubleDelta), + ErrNumSamples UInt32 +) ENGINE = SummingMergeTree ((NumSamples)) +PARTITION BY + toYYYYMMDD (Timestamp) +ORDER BY + ( + ServiceId, + Timestamp, + CallStackHash, + CallStackParent + ); +CREATE MATERIALIZED VIEW IF NOT EXISTS flamedb.samples_1day_all_mv to flamedb.samples_1day_all AS +SELECT + toStartOfDay (Timestamp) AS Timestamp, + ServiceId, + CallStackHash, + any (CallStackName) as CallStackName, + any (CallStackParent) as CallStackParent, + sum(NumSamples) AS NumSamples, + sum(ErrNumSamples) AS ErrNumSamples, + anyLast (InsertionTimestamp) as InsertionTimestamp +FROM + flamedb.samples +GROUP BY + ServiceId, + CallStackHash, + Timestamp; -- create 1day aggregated table -CREATE TABLE IF NOT EXISTS flamedb.samples_1day -( - Timestamp DateTime('UTC') CODEC(DoubleDelta), +CREATE TABLE IF NOT EXISTS flamedb.samples_1day ( + Timestamp DateTime ('UTC') CODEC (DoubleDelta), ServiceId UInt32, - InstanceType LowCardinality(String), - ContainerEnvName LowCardinality(String), - HostName LowCardinality(String), - ContainerName LowCardinality(String), - CallStackHash UInt64, - CallStackName String CODEC (ZSTD), - CallStackParent UInt64, - InsertionTimestamp DateTime('UTC') CODEC(DoubleDelta), - NumSamples UInt64 CODEC(DoubleDelta), - ErrNumSamples UInt32 -) ENGINE = SummingMergeTree((NumSamples)) - PARTITION BY toYYYYMMDD(Timestamp) - ORDER BY (ServiceId, ContainerEnvName, InstanceType, HostName, ContainerName, - Timestamp, CallStackHash, CallStackParent); - - -CREATE MATERIALIZED VIEW IF NOT EXISTS - flamedb.samples_1day_mv TO flamedb.samples_1day -AS -SELECT toStartOfDay(Timestamp) AS Timestamp, - ServiceId, - ContainerEnvName, - InstanceType, - HostName, - ContainerName, - CallStackHash, - any(CallStackName) as CallStackName, - any(CallStackParent) as CallStackParent, - sum(NumSamples) AS NumSamples, - sum(ErrNumSamples) AS ErrNumSamples, - anyLast(InsertionTimestamp) as InsertionTimestamp -FROM flamedb.samples -GROUP BY ServiceId, InstanceType, ContainerEnvName, HostName, ContainerName, - CallStackHash, Timestamp; + InstanceType LowCardinality (String), + ContainerEnvName LowCardinality (String), + HostName LowCardinality (String), + ContainerName LowCardinality (String), + CallStackHash UInt64, + CallStackName String CODEC (ZSTD), + CallStackParent UInt64, + InsertionTimestamp DateTime ('UTC') CODEC (DoubleDelta), + NumSamples UInt64 CODEC (DoubleDelta), + ErrNumSamples UInt32 +) ENGINE = SummingMergeTree ((NumSamples)) +PARTITION BY + toYYYYMMDD (Timestamp) +ORDER BY + ( + ServiceId, + ContainerEnvName, + InstanceType, + HostName, + ContainerName, + Timestamp, + CallStackHash, + CallStackParent + ); +CREATE MATERIALIZED VIEW IF NOT EXISTS flamedb.samples_1day_mv TO flamedb.samples_1day AS +SELECT + toStartOfDay (Timestamp) AS Timestamp, + ServiceId, + ContainerEnvName, + InstanceType, + HostName, + ContainerName, + CallStackHash, + any (CallStackName) as CallStackName, + any (CallStackParent) as CallStackParent, + sum(NumSamples) AS NumSamples, + sum(ErrNumSamples) AS ErrNumSamples, + anyLast (InsertionTimestamp) as InsertionTimestamp +FROM + flamedb.samples +GROUP BY + ServiceId, + InstanceType, + ContainerEnvName, + HostName, + ContainerName, + CallStackHash, + Timestamp; -- create local table CREATE TABLE IF NOT EXISTS flamedb.samples_1min ( - Timestamp DateTime('UTC') CODEC(DoubleDelta), + Timestamp DateTime ('UTC') CODEC (DoubleDelta), ServiceId UInt32, - InstanceType LowCardinality(String), - ContainerEnvName LowCardinality(String), - HostName LowCardinality(String), - ContainerName LowCardinality(String), - NumSamples UInt64 CODEC(DoubleDelta), + InstanceType LowCardinality (String), + ContainerEnvName LowCardinality (String), + HostName LowCardinality (String), + ContainerName LowCardinality (String), + NumSamples UInt64 CODEC (DoubleDelta), ErrNumSamples UInt64, HostNameHash UInt32, ContainerNameHash UInt32, - InsertionTimestamp DateTime('UTC') CODEC(DoubleDelta) -) ENGINE = SummingMergeTree((NumSamples)) - PARTITION BY toYYYYMMDD(Timestamp) - ORDER BY (ServiceId, ContainerEnvName, InstanceType, ContainerNameHash, HostNameHash, Timestamp); + InsertionTimestamp DateTime ('UTC') CODEC (DoubleDelta) +) ENGINE = SummingMergeTree ((NumSamples)) +PARTITION BY + toYYYYMMDD (Timestamp) +ORDER BY + ( + ServiceId, + ContainerEnvName, + InstanceType, + ContainerNameHash, + HostNameHash, + Timestamp + ); -- create mv -CREATE MATERIALIZED VIEW IF NOT EXISTS flamedb.samples_1min_mv TO - flamedb.samples_1min -AS SELECT toStartOfMinute(Timestamp) AS - Timestamp, - ServiceId, - InstanceType, - ContainerEnvName, - HostName, - ContainerName, - sum(NumSamples) AS NumSamples, - sum(ErrNumSamples) AS ErrNumSamples, - HostNameHash, - ContainerNameHash, - anyLast(InsertionTimestamp) as InsertionTimestamp - FROM flamedb.samples WHERE CallStackParent = 0 - GROUP BY ServiceId, InstanceType, ContainerEnvName, HostName, ContainerName, HostNameHash, ContainerNameHash, Timestamp; - -ALTER TABLE flamedb.samples - MODIFY TTL "Timestamp" + INTERVAL 30 DAY; - -ALTER TABLE flamedb.samples_1hour - MODIFY TTL "Timestamp" + INTERVAL 30 DAY; - -ALTER TABLE flamedb.samples_1hour_all - MODIFY TTL "Timestamp" + INTERVAL 30 DAY; - -ALTER TABLE flamedb.samples_1day_all - MODIFY TTL "Timestamp" + INTERVAL 30 DAY; - -ALTER TABLE flamedb.samples_1day - MODIFY TTL "Timestamp" + INTERVAL 365 DAY; +CREATE MATERIALIZED VIEW IF NOT EXISTS flamedb.samples_1min_mv TO flamedb.samples_1min AS +SELECT + toStartOfMinute (Timestamp) AS Timestamp, + ServiceId, + InstanceType, + ContainerEnvName, + HostName, + ContainerName, + sum(NumSamples) AS NumSamples, + sum(ErrNumSamples) AS ErrNumSamples, + HostNameHash, + ContainerNameHash, + anyLast (InsertionTimestamp) as InsertionTimestamp +FROM + flamedb.samples +WHERE + CallStackParent = 0 +GROUP BY + ServiceId, + InstanceType, + ContainerEnvName, + HostName, + ContainerName, + HostNameHash, + ContainerNameHash, + Timestamp; + +ALTER TABLE flamedb.samples MODIFY TTL "Timestamp" + INTERVAL 30 DAY; + +ALTER TABLE flamedb.samples_1hour MODIFY TTL "Timestamp" + INTERVAL 30 DAY; + +ALTER TABLE flamedb.samples_1hour_all MODIFY TTL "Timestamp" + INTERVAL 30 DAY; + +ALTER TABLE flamedb.samples_1day_all MODIFY TTL "Timestamp" + INTERVAL 30 DAY; + +ALTER TABLE flamedb.samples_1day MODIFY TTL "Timestamp" + INTERVAL 365 DAY; diff --git a/src/gprofiler_indexer/sql/migrations/20250310_1301_add_new_metrics_columns.sql b/src/gprofiler_indexer/sql/migrations/20250310_1301_add_new_metrics_columns.sql new file mode 100644 index 00000000..e73c2cc6 --- /dev/null +++ b/src/gprofiler_indexer/sql/migrations/20250310_1301_add_new_metrics_columns.sql @@ -0,0 +1,17 @@ +ALTER TABLE flamedb.metrics +ADD COLUMN CPUFrequency Nullable (Float64) AFTER MemoryAverageUsedPercent; + +ALTER TABLE flamedb.metrics +ADD COLUMN CPUCPI Nullable (Float64) AFTER CPUFrequency; + +ALTER TABLE flamedb.metrics +ADD COLUMN CPUTMAFrontEndBound Nullable (Float64) AFTER CPUCPI; + +ALTER TABLE flamedb.metrics +ADD COLUMN CPUTMABackendBound Nullable (Float64) AFTER CPUTMAFrontEndBound; + +ALTER TABLE flamedb.metrics +ADD COLUMN CPUTMABadSpec Nullable (Float64) AFTER CPUTMABackendBound; + +ALTER TABLE flamedb.metrics +ADD COLUMN CPUTMARetiring Nullable (Float64) AFTER CPUTMABadSpec;