25.85s of 29.47s total (87.72%) Dropped 313 nodes (cum <= 0.15s) flat flat% sum% cum cum% 0 0% 0% 26.74s 90.74% runtime.goexit 0 0% 0% 17.61s 59.76% github.com/prometheus/prometheus/promql.(*Engine).exec 0 0% 0% 17.61s 59.76% github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt 0 0% 0% 17.61s 59.76% github.com/prometheus/prometheus/promql.(*query).Exec 0.02s 0.068% 0.068% 17.12s 58.09% github.com/prometheus/prometheus/storage/local/index.(*LevelDB).Get 0.03s 0.1% 0.17% 16.13s 54.73% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).Get 0.02s 0.068% 0.24% 16.04s 54.43% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).get 0 0% 0.24% 15.44s 52.39% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*version).get 0.02s 0.068% 0.31% 15.44s 52.39% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*version).walkOverlapping 0.01s 0.034% 0.34% 15.23s 51.68% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*version).get.func1 0.02s 0.068% 0.41% 15.18s 51.51% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*tOps).find 0.02s 0.068% 0.48% 15.01s 50.93% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).Find 0.08s 0.27% 0.75% 14.99s 50.87% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).find 0.03s 0.1% 0.85% 14.15s 48.01% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/cache.(*Cache).Get 0.03s 0.1% 0.95% 14.06s 47.71% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/cache.(*CacheGetter).Get 0 0% 0.95% 13.75s 46.66% github.com/prometheus/prometheus/promql.Inspect 0 0% 0.95% 13.75s 46.66% github.com/prometheus/prometheus/promql.Walk 0 0% 0.95% 13.75s 46.66% github.com/prometheus/prometheus/promql.inspector.Visit 0 0% 0.95% 13.56s 46.01% github.com/prometheus/prometheus/promql.(*Analyzer).Analyze 0 0% 0.95% 13.56s 46.01% github.com/prometheus/prometheus/promql.(*Analyzer).Analyze.func2 0 0% 0.95% 13.56s 46.01% github.com/prometheus/prometheus/storage/local.(*memorySeriesStorage).MetricsForLabelMatchers 0.02s 0.068% 1.02% 13.51s 45.84% github.com/prometheus/prometheus/storage/local.(*memorySeriesStorage).MetricForFingerprint 0 0% 1.02% 13.41s 45.50% github.com/prometheus/prometheus/storage/local.(*persistence).archivedMetric 0 0% 1.02% 13.41s 45.50% github.com/prometheus/prometheus/storage/local/index.(*FingerprintMetricIndex).Lookup 0.03s 0.1% 1.12% 13.40s 45.47% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).readRawBlock 0 0% 1.12% 12.66s 42.96% github.com/prometheus/prometheus/rules.(*Manager).runIteration.func1 0 0% 1.12% 11.57s 39.26% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).readBlockCached 0.01s 0.034% 1.15% 11.13s 37.77% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).readBlock 0.01s 0.034% 1.19% 11.03s 37.43% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).readBlockCached.func1 0 0% 1.19% 9.35s 31.73% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).getIndexBlock 6.44s 21.85% 23.04% 7.66s 25.99% github.com/prometheus/prometheus/vendor/github.com/golang/snappy.Decode 0 0% 23.04% 6.61s 22.43% github.com/prometheus/prometheus/rules.(*RecordingRule).eval 0 0% 23.04% 6.61s 22.43% github.com/prometheus/prometheus/rules.RecordingRule.eval 0 0% 23.04% 6.05s 20.53% github.com/prometheus/prometheus/rules.(*AlertingRule).eval 0.13s 0.44% 23.48% 5.87s 19.92% runtime.systemstack 2.65s 8.99% 32.47% 5.85s 19.85% runtime.scanobject 0 0% 32.47% 5.43s 18.43% runtime.gcBgMarkWorker 0.27s 0.92% 33.39% 5s 16.97% runtime.gcDrainUntilPreempt 0 0% 33.39% 4.95s 16.80% github.com/prometheus/prometheus/util/httputil.(*CompressionHandler).ServeHTTP 0 0% 33.39% 4.95s 16.80% github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP 0 0% 33.39% 4.95s 16.80% github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP 0 0% 33.39% 4.95s 16.80% github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1 0 0% 33.39% 4.95s 16.80% github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP 0 0% 33.39% 4.95s 16.80% github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.handle.func1 0 0% 33.39% 4.95s 16.80% github.com/prometheus/prometheus/web/api/v1.(*API).(github.com/prometheus/prometheus/web/api/v1.query)-fm 0 0% 33.39% 4.95s 16.80% github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1 0 0% 33.39% 4.95s 16.80% github.com/prometheus/prometheus/web/api/v1.(*API).query 0 0% 33.39% 4.95s 16.80% net/http.(*conn).serve 0 0% 33.39% 4.95s 16.80% net/http.(Handler).ServeHTTP-fm 0 0% 33.39% 4.95s 16.80% net/http.HandlerFunc.ServeHTTP 0 0% 33.39% 4.95s 16.80% net/http.serverHandler.ServeHTTP 0 0% 33.39% 4.03s 13.67% github.com/prometheus/prometheus/promql.(*Analyzer).Prepare 0 0% 33.39% 3.83s 13.00% github.com/prometheus/prometheus/storage/local.(*memorySeriesPreloader).PreloadRange 0 0% 33.39% 3.83s 13.00% github.com/prometheus/prometheus/storage/local.(*memorySeriesStorage).preloadChunksForRange 0.45s 1.53% 34.92% 3.83s 13.00% runtime.mallocgc 0 0% 34.92% 3.73s 12.66% github.com/prometheus/prometheus/storage/local.(*persistence).hasArchivedMetric 0 0% 34.92% 3.73s 12.66% github.com/prometheus/prometheus/storage/local/index.(*FingerprintTimeRangeIndex).Lookup 0.02s 0.068% 34.98% 3.24s 10.99% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).Get 0.01s 0.034% 35.02% 3.21s 10.89% runtime.newarray 0.05s 0.17% 35.19% 3.15s 10.69% runtime.makeslice 0.01s 0.034% 35.22% 2.63s 8.92% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).getFilterBlock 0.02s 0.068% 35.29% 2.62s 8.89% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).readFilterBlockCached 0 0% 35.29% 2.46s 8.35% runtime.mcall 0.04s 0.14% 35.43% 2.42s 8.21% runtime.schedule 0.02s 0.068% 35.49% 2.35s 7.97% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).readFilterBlock 0 0% 35.49% 2.35s 7.97% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).readFilterBlockCached.func1 0 0% 35.49% 2.28s 7.74% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Reader).getDataIter 0.02s 0.068% 35.56% 1.94s 6.58% runtime.bgsweep 0.30s 1.02% 36.58% 1.93s 6.55% runtime.findrunnable 0.01s 0.034% 36.61% 1.91s 6.48% runtime.gosweepone 0.07s 0.24% 36.85% 1.89s 6.41% runtime.sweepone 0 0% 36.85% 1.83s 6.21% runtime.gosweepone.func1 0.79s 2.68% 39.53% 1.78s 6.04% runtime.mSpan_Sweep 1.08s 3.66% 43.20% 1.77s 6.01% runtime.greyobject 1.67s 5.67% 48.86% 1.67s 5.67% runtime.memclr 1.50s 5.09% 53.95% 1.50s 5.09% runtime.heapBitsForObject 0.02s 0.068% 54.02% 1.49s 5.06% runtime.mHeap_Alloc 1.44s 4.89% 58.91% 1.48s 5.02% syscall.Syscall6 0.01s 0.034% 58.94% 1.46s 4.95% os.(*File).ReadAt 0.02s 0.068% 59.01% 1.45s 4.92% os.(*File).pread 1.44s 4.89% 63.90% 1.44s 4.89% runtime.memmove 0 0% 63.90% 1.43s 4.85% syscall.Pread 0 0% 63.90% 1.40s 4.75% runtime.largeAlloc 0 0% 63.90% 1.40s 4.75% runtime.mallocgc.func3 0 0% 63.90% 1.39s 4.72% runtime.goschedImpl 0 0% 63.90% 1.38s 4.68% runtime.gosched_m 0 0% 63.90% 1.07s 3.63% runtime.park_m 0 0% 63.90% 1s 3.39% runtime.gcAssistAlloc 0 0% 63.90% 1s 3.39% runtime.gcAssistAlloc.func1 0.02s 0.068% 63.96% 1s 3.39% runtime.gcDrainN 0 0% 63.96% 0.95s 3.22% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.CRC.Update 0 0% 63.96% 0.95s 3.22% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewCRC 0 0% 63.96% 0.95s 3.22% hash/crc32.Update 0.95s 3.22% 67.19% 0.95s 3.22% hash/crc32.castagnoliSSE42 0 0% 67.19% 0.95s 3.22% hash/crc32.updateCastagnoli 0.01s 0.034% 67.22% 0.93s 3.16% github.com/prometheus/prometheus/storage/local/codable.(*Metric).UnmarshalBinary 0.52s 1.76% 68.99% 0.93s 3.16% runtime.heapBitsSweepSpan 0.18s 0.61% 69.60% 0.93s 3.16% runtime.runqsteal 0.01s 0.034% 69.63% 0.89s 3.02% github.com/prometheus/prometheus/storage/local/codable.(*Metric).UnmarshalFromReader 0.86s 2.92% 72.55% 0.86s 2.92% runtime.futex 0.40s 1.36% 73.91% 0.75s 2.54% runtime.runqgrab 0.03s 0.1% 74.01% 0.72s 2.44% github.com/prometheus/prometheus/storage/local/codable.decodeString 0 0% 74.01% 0.68s 2.31% github.com/prometheus/prometheus/storage/local.(*memorySeriesStorage).loop 0.09s 0.31% 74.31% 0.65s 2.21% runtime.newobject 0.01s 0.034% 74.35% 0.59s 2.00% github.com/prometheus/prometheus/storage/local.(*memorySeriesStorage).maintainMemorySeries 0 0% 74.35% 0.58s 1.97% runtime.stopm 0.09s 0.31% 74.65% 0.57s 1.93% runtime.heapBits.setMarked 0 0% 74.65% 0.56s 1.90% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionTransact 0 0% 74.65% 0.56s 1.90% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction 0 0% 74.65% 0.56s 1.90% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tableAutoCompaction 0 0% 74.65% 0.56s 1.90% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tableCompaction 0 0% 74.65% 0.56s 1.90% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*tableCompactionBuilder).run 0.02s 0.068% 74.72% 0.52s 1.76% runtime.gcDrain 0.07s 0.24% 74.96% 0.50s 1.70% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*iComparer).Compare 0.48s 1.63% 76.59% 0.48s 1.63% runtime.atomicor8 0 0% 76.59% 0.47s 1.59% runtime.futexsleep 0 0% 76.59% 0.45s 1.53% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*blockIter).Seek 0 0% 76.59% 0.45s 1.53% runtime.notesleep 0 0% 76.59% 0.44s 1.49% github.com/prometheus/prometheus/storage/local.(*memorySeriesStorage).writeMemorySeries 0.15s 0.51% 77.10% 0.41s 1.39% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/memdb.(*DB).findGE 0.36s 1.22% 78.32% 0.41s 1.39% runtime.mSpan_Sweep.func1 0 0% 78.32% 0.41s 1.39% runtime.resetspinning 0 0% 78.32% 0.41s 1.39% runtime.wakep 0 0% 78.32% 0.40s 1.36% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/memdb.(*DB).Find 0 0% 78.32% 0.40s 1.36% runtime.startm 0.05s 0.17% 78.49% 0.39s 1.32% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/cache.(*lru).Promote 0 0% 78.49% 0.39s 1.32% runtime.futexwakeup 0.03s 0.1% 78.59% 0.39s 1.32% sort.Search 0 0% 78.59% 0.37s 1.26% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*tableCompactionBuilder).appendKV 0 0% 78.59% 0.37s 1.26% runtime.backgroundgc 0 0% 78.59% 0.37s 1.26% runtime.gc 0 0% 78.59% 0.37s 1.26% runtime.notewakeup 0 0% 78.59% 0.36s 1.22% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*tWriter).append 0 0% 78.59% 0.36s 1.22% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Writer).Append 0.05s 0.17% 78.76% 0.35s 1.19% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/comparer.(*bytesComparer).Compare 0.08s 0.27% 79.03% 0.33s 1.12% runtime.deferproc 0.30s 1.02% 80.05% 0.32s 1.09% runtime.atomicload 0.06s 0.2% 80.25% 0.30s 1.02% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/comparer.bytesComparer.Compare 0.18s 0.61% 80.86% 0.30s 1.02% runtime.deferreturn 0 0% 80.86% 0.29s 0.98% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/filter.bloomHash 0.01s 0.034% 80.90% 0.29s 0.98% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.Hash 0.26s 0.88% 81.78% 0.26s 0.88% runtime.usleep 0.26s 0.88% 82.66% 0.26s 0.88% sync/atomic.AddUint32 0 0% 82.66% 0.25s 0.85% github.com/prometheus/prometheus/storage/local.(*persistence).persistChunks 0.11s 0.37% 83.03% 0.24s 0.81% runtime.(*gcWork).tryGet 0.14s 0.48% 83.51% 0.24s 0.81% runtime.mapassign1 0.01s 0.034% 83.54% 0.24s 0.81% runtime.markroot 0.23s 0.78% 84.32% 0.23s 0.78% runtime.cmpbody 0.03s 0.1% 84.42% 0.23s 0.78% runtime.deferproc.func1 0 0% 84.42% 0.23s 0.78% runtime.mCache_Refill 0.03s 0.1% 84.53% 0.23s 0.78% runtime.mCentral_CacheSpan 0 0% 84.53% 0.23s 0.78% runtime.parfordo 0.20s 0.68% 85.21% 0.23s 0.78% syscall.Syscall 0 0% 85.21% 0.22s 0.75% runtime.gc.func1 0 0% 85.21% 0.22s 0.75% runtime.mallocgc.func2 0 0% 85.21% 0.21s 0.71% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*block).seek 0.06s 0.2% 85.41% 0.21s 0.71% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*blockIter).Next 0.07s 0.24% 85.65% 0.21s 0.71% runtime.scang 0.09s 0.31% 85.95% 0.20s 0.68% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/cache.(*mBucket).get 0.11s 0.37% 86.33% 0.20s 0.68% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*block).seek.func1 0 0% 86.33% 0.20s 0.68% runtime.gcscan_m 0 0% 86.33% 0.20s 0.68% runtime.mProf_Malloc 0 0% 86.33% 0.20s 0.68% runtime.profilealloc 0.01s 0.034% 86.36% 0.19s 0.64% github.com/prometheus/prometheus/promql.(*Analyzer).Prepare.func2 0.01s 0.034% 86.39% 0.19s 0.64% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.tFiles.searchMax 0.03s 0.1% 86.49% 0.18s 0.61% github.com/prometheus/prometheus/storage/local/codable.putBuf 0 0% 86.49% 0.18s 0.61% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/cache.(*Handle).Release 0 0% 86.49% 0.18s 0.61% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/iterator.(*indexedIterator).Next 0 0% 86.49% 0.18s 0.61% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/iterator.(*mergedIterator).Next 0.01s 0.034% 86.53% 0.17s 0.58% runtime.callers 0.05s 0.17% 86.70% 0.17s 0.58% runtime.gentraceback 0.16s 0.54% 87.24% 0.17s 0.58% runtime.newdefer 0.02s 0.068% 87.31% 0.17s 0.58% sync.(*Mutex).Lock 0.01s 0.034% 87.34% 0.16s 0.54% github.com/prometheus/prometheus/storage/local.(*seriesMap).get 0.05s 0.17% 87.51% 0.16s 0.54% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.tFiles.searchMax.func1 0 0% 87.51% 0.16s 0.54% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/cache.(*Node).unrefLocked 0 0% 87.51% 0.16s 0.54% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Writer).finishBlock 0.01s 0.034% 87.55% 0.16s 0.54% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*filterWriter).add 0 0% 87.55% 0.16s 0.54% os.(*File).Write 0 0% 87.55% 0.16s 0.54% os.(*File).write 0 0% 87.55% 0.16s 0.54% runtime.callers.func1 0.01s 0.034% 87.58% 0.16s 0.54% runtime.convT2E 0 0% 87.58% 0.16s 0.54% runtime.mHeap_Alloc.func1 0 0% 87.58% 0.16s 0.54% runtime.mHeap_Alloc_m 0 0% 87.58% 0.16s 0.54% syscall.Write 0 0% 87.58% 0.16s 0.54% syscall.write 0.03s 0.1% 87.68% 0.15s 0.51% encoding/binary.Read 0 0% 87.68% 0.15s 0.51% github.com/prometheus/prometheus/storage/local.(*memorySeriesStorage).writeMemorySeries.func1 0 0% 87.68% 0.15s 0.51% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*iFilterGenerator).Add 0 0% 87.68% 0.15s 0.51% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.iFilterGenerator.Add 0 0% 87.68% 0.15s 0.51% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/filter.(*bloomFilterGenerator).Add 0 0% 87.68% 0.15s 0.51% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*Writer).writeBlock 0.01s 0.034% 87.72% 0.15s 0.51% github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/table.(*filterBlock).contains