|
|
<!DOCTYPE html>
|
|
|
<html lang="en-us">
|
|
|
<head><meta charset="utf-8">
|
|
|
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
|
|
<script type="application/ld+json">
|
|
|
{
|
|
|
"@context" : "http://schema.org",
|
|
|
"@type" : "BlogPosting",
|
|
|
"mainEntityOfPage": {
|
|
|
"@type": "WebPage",
|
|
|
"@id": "https:\/\/h.cowbay.org"
|
|
|
},
|
|
|
"articleSection" : "post",
|
|
|
"name" : "[筆記] 測試 postgresql 的pg_prewarm 對效能的影響 \/ test pg_prewarm in postgresql 11",
|
|
|
"headline" : "[筆記] 測試 postgresql 的pg_prewarm 對效能的影響 \/ test pg_prewarm in postgresql 11",
|
|
|
"description" : "\x3cp\x3e老闆提到想要把新系統的 postgresql 資料庫都撈到記憶體裡面\x3c\/p\x3e\n\n\x3cp\x3e但是否決了我提出的ramdisk 作法(因為當機的話，資料就沒了)\x3c\/p\x3e\n\n\x3cp\x3e在找資料的時候，發現了這個postgresql 的 pg_prewarm extension\x3c\/p\x3e\n\n\x3cp\x3e好像有點意思？就來測試看看吧！\x3c\/p\x3e\n\n\x3cp\x3e只是目前還不知道該怎麼解讀測試的數據就是了\x26hellip;\x3c\/p\x3e\n\n\x3cp\x3e幹！林北真的不是 DBA 啦 =.=\x3c\/p\x3e",
|
|
|
"inLanguage" : "en",
|
|
|
"author" : "Eric Chang",
|
|
|
"creator" : "Eric Chang",
|
|
|
"publisher": "Eric Chang",
|
|
|
"accountablePerson" : "Eric Chang",
|
|
|
"copyrightHolder" : "Eric Chang",
|
|
|
"copyrightYear" : "2019",
|
|
|
"datePublished": "2019-12-20 14:31:42 \x2b0800 CST",
|
|
|
"dateModified" : "2019-12-20 14:31:42 \x2b0800 CST",
|
|
|
"url" : "https:\/\/h.cowbay.org\/post\/test-pg_prewarm\/",
|
|
|
"wordCount" : "1314",
|
|
|
"image" : "https://h.cowbay.orghttps://h.cowbay.org/images/post-default-9.jpg"",
|
|
|
"keywords" : [ ""postgresql"","Blog" ]
|
|
|
}
|
|
|
</script>
|
|
|
|
|
|
|
|
|
<title>[筆記] 測試 postgresql 的pg_prewarm 對效能的影響 / test pg_prewarm in postgresql 11 </title>
|
|
|
|
|
|
|
|
|
<meta name="description" content="some articles about job,food,passion sisters" />
|
|
|
|
|
|
|
|
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
|
<meta name="robots" content="all,follow">
|
|
|
<meta name="googlebot" content="index,follow,snippet,archive">
|
|
|
<link rel="stylesheet" id="ct-tracks-google-fonts-css" href="https://fonts.googleapis.com/css?family=Raleway%3A400%2C700&subset=latin%2Clatin-ext&ver=4.7.2" type="text/css" media="all">
|
|
|
|
|
|
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.3.1/css/all.css" integrity="sha384-mzrmE5qonljUremFsqc01SB46JvROS7bZs3IO2EmfFsd15uHvIt+Y8vEf7N7fWAU" crossorigin="anonymous">
|
|
|
|
|
|
<link href="https://h.cowbay.org/css/style.css?v=1626744134" rel="stylesheet" id="theme-stylesheet" type='text/css' media='all'>
|
|
|
|
|
|
<link href="https://h.cowbay.org/css/custom.css?v=1626744134" rel="stylesheet" type='text/css' media='all'>
|
|
|
<link rel="shortcut icon" href="https://h.cowbay.org/img/favicon.ico" type="image/x-icon">
|
|
|
<link rel="icon" href="https://h.cowbay.org/img/favicon.ico" type="image/x-icon">
|
|
|
|
|
|
|
|
|
<script type="application/javascript">
|
|
|
var doNotTrack = false;
|
|
|
if (!doNotTrack) {
|
|
|
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
|
|
|
ga('create', 'UA-138954876-1', 'auto');
|
|
|
|
|
|
ga('send', 'pageview');
|
|
|
}
|
|
|
</script>
|
|
|
<script async src='https://www.google-analytics.com/analytics.js'></script>
|
|
|
|
|
|
</head>
|
|
|
|
|
|
|
|
|
<body class="post-template-default single single-post single-format-standard ct-body singular singular-post not-front standard">
|
|
|
|
|
|
<div id="overflow-container" class="overflow-container">
|
|
|
<a class="skip-content" href="#main">Skip to content</a>
|
|
|
<header id="site-header" class="site-header" role="banner">
|
|
|
<div class='top-navigation'>
|
|
|
<div class='container'>
|
|
|
|
|
|
<div id="menu-secondary" class="menu-container menu-secondary" role="navigation">
|
|
|
<button id="toggle-secondary-navigation" class="toggle-secondary-navigation"><i class="fas fa-plus"></i></button>
|
|
|
|
|
|
<div class="menu">
|
|
|
|
|
|
<ul id="menu-secondary-items" class="menu-secondary-items">
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/"></a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/ansible">ansible</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/linux">linux</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/proxmox">proxmox</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/ps">ps</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/%E7%A2%8E%E5%BF%B5">碎念</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/%E7%AD%86%E8%A8%98">筆記</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/%E7%BE%A4%E6%9A%89">群暉</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/%E9%9B%9C%E5%BF%B5">雜念</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
</ul>
|
|
|
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
<ul class="social-media-icons">
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20Social%20profile%20url%20in%20facebook" data-animate-hover="pulse" class="facebook" target="_blank">
|
|
|
<i class="fab fa-facebook-square" title="facebook"></i>
|
|
|
<span class="screen-reader-text">facebook</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20googleplus" data-animate-hover="pulse" class="gplus" target="_blank">
|
|
|
<i class="fab fa-google-plus-g" title="googleplus"></i>
|
|
|
<span class="screen-reader-text">googleplus</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="chang0206" data-animate-hover="pulse" class="twitter" target="_blank">
|
|
|
<i class="fab fa-twitter-square" title="twitter"></i>
|
|
|
<span class="screen-reader-text">twitter</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="chang0206" data-animate-hover="pulse" class="instagram" target="_blank">
|
|
|
<i class="fab fa-instagram" title="instagram"></i>
|
|
|
<span class="screen-reader-text">instagram</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="mailto:mc@hotshraingmy.info" data-animate-hover="pulse" class="email">
|
|
|
<i class="fas fa-envelope" title="email"></i>
|
|
|
<span class="screen-reader-text">email</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20linkedin" data-animate-hover="pulse" class="linkedin" target="_blank">
|
|
|
<i class="fab fa-linkedin-in" title="linkedin"></i>
|
|
|
<span class="screen-reader-text">linkedin</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20stackoverflow" data-animate-hover="pulse" class="stackoverflow" target="_blank">
|
|
|
<i class="fab fa-stack-overflow" title="stackoverflow"></i>
|
|
|
<span class="screen-reader-text">stackoverflow</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="changchichung" data-animate-hover="pulse" class="github" target="_blank">
|
|
|
<i class="fab fa-github" title="github"></i>
|
|
|
<span class="screen-reader-text">github</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20pinterest" data-animate-hover="pulse" class="pinterest" target="_blank">
|
|
|
<i class="fab fa-pinterest" title="pinterest"></i>
|
|
|
<span class="screen-reader-text">pinterest</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="https://h.cowbay.org/index.xml" data-animate-hover="pulse" class="rss" target="_blank">
|
|
|
<i class="fas fa-rss" title="rss"></i>
|
|
|
<span class="screen-reader-text">rss</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
</ul></div>
|
|
|
|
|
|
</div>
|
|
|
|
|
|
<div class="container">
|
|
|
<div id="title-info" class="title-info">
|
|
|
<div id='site-title' class='site-title'>
|
|
|
|
|
|
<a href="/"> MC部落 </a>
|
|
|
</div>
|
|
|
</div>
|
|
|
<button id="toggle-navigation" class="toggle-navigation">
|
|
|
<i class="fas fa-bars"></i>
|
|
|
</button>
|
|
|
|
|
|
<div id="menu-primary-tracks" class="menu-primary-tracks"></div>
|
|
|
<div id="menu-primary" class="menu-container menu-primary" role="navigation">
|
|
|
|
|
|
<p class="site-description">What’s the Worst That Could Happen?</p>
|
|
|
|
|
|
|
|
|
<div class="menu">
|
|
|
<ul id="menu-primary-items" class="menu-primary-items">
|
|
|
|
|
|
|
|
|
<li class='menu-item menu-item-type-custom menu-item-object-custom '>
|
|
|
<a href="https://h.cowbay.org/">Home</a>
|
|
|
|
|
|
</li>
|
|
|
|
|
|
<li class='menu-item menu-item-type-post_type menu-item-object-page '>
|
|
|
<a href="https://h.cowbay.org/about/">About</a>
|
|
|
|
|
|
</li>
|
|
|
|
|
|
<li class='menu-item menu-item-type-post_type menu-item-object-page '>
|
|
|
<a href="https://h.cowbay.org/contact/">Get in touch</a>
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
</header>
|
|
|
|
|
|
<div id="main" class="main" role="main">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div id="loop-container" class="loop-container">
|
|
|
|
|
|
<div class="post type-post status-publish format-standard has-post-thumbnail hentry category-design tag-design tag-standard-2 tag-tagalicious tag-travel entry full-without-featured odd excerpt-1">
|
|
|
|
|
|
<div class='featured-image lazy lazy-bg-image' data-background="https://h.cowbay.org/images/post-default-9.jpg">
|
|
|
</div>
|
|
|
|
|
|
<div class="entry-meta">
|
|
|
<span class="date">20 December</span> <span> / </span>
|
|
|
|
|
|
<span class="author">
|
|
|
<a href="https://github.com/changchichung" title="Posts by Eric Chang" rel="author">Eric Chang</a>
|
|
|
</span>
|
|
|
|
|
|
|
|
|
|
|
|
<span class="category">
|
|
|
<span> / </span>
|
|
|
|
|
|
<a href="/categories/%E7%AD%86%E8%A8%98">筆記</a>
|
|
|
</span>
|
|
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
<div class='entry-header'>
|
|
|
<h1 class='entry-title'> [筆記] 測試 postgresql 的pg_prewarm 對效能的影響 / test pg_prewarm in postgresql 11</h1>
|
|
|
</div>
|
|
|
<div class="entry-container">
|
|
|
<div class="entry-content">
|
|
|
<article>
|
|
|
<p>老闆提到想要把新系統的 postgresql 資料庫都撈到記憶體裡面</p>
|
|
|
|
|
|
<p>但是否決了我提出的ramdisk 作法(因為當機的話，資料就沒了)</p>
|
|
|
|
|
|
<p>在找資料的時候，發現了這個postgresql 的 pg_prewarm extension</p>
|
|
|
|
|
|
<p>好像有點意思？就來測試看看吧！</p>
|
|
|
|
|
|
<p>只是目前還不知道該怎麼解讀測試的數據就是了…</p>
|
|
|
|
|
|
<p>幹！林北真的不是 DBA 啦 =.=</p>
|
|
|
|
|
|
<p>安裝系統、postgresql 資料庫什麼的就不提了，那不是這次的重點</p>
|
|
|
|
|
|
<h4 id="修改-postgresql-conf">修改 postgresql.conf</h4>
|
|
|
|
|
|
<p>編輯postgresql.conf，開啟平行處理以及設定可用記憶體容量</p>
|
|
|
|
|
|
<p>這台測試機的環境是一台三代i7 , 24G RAM , 240G SSD，安裝debian 10(buster)</p>
|
|
|
|
|
|
<pre><code># load libiriaes
|
|
|
# 其實這次不會用到pg_stat_statements ，不過出於習慣，還是加入開機自動載入吧
|
|
|
shared_preload_libraries = 'pg_stat_statements'
|
|
|
|
|
|
#------------------------------------------------------------------------------
|
|
|
# CUSTOMIZED OPTIONS
|
|
|
#------------------------------------------------------------------------------
|
|
|
|
|
|
max_connections = 20
|
|
|
shared_buffers = 6GB
|
|
|
effective_cache_size = 18GB
|
|
|
maintenance_work_mem = 1536MB
|
|
|
checkpoint_completion_target = 0.7
|
|
|
wal_buffers = 16MB
|
|
|
default_statistics_target = 100
|
|
|
random_page_cost = 1.1
|
|
|
effective_io_concurrency = 200
|
|
|
work_mem = 78643kB
|
|
|
min_wal_size = 1GB
|
|
|
max_wal_size = 2GB
|
|
|
max_worker_processes = 8
|
|
|
max_parallel_workers_per_gather = 4
|
|
|
max_parallel_workers = 8
|
|
|
</code></pre>
|
|
|
|
|
|
<p>重新啟動postgresql ，準備開始測試囉！</p>
|
|
|
|
|
|
<p>轉換成 postgres 身份後，進入 psql</p>
|
|
|
|
|
|
<h4 id="建立測試資料庫">建立測試資料庫</h4>
|
|
|
|
|
|
<pre><code>postgres=# create database test;
|
|
|
CREATE DATABASE
|
|
|
postgres=#
|
|
|
</code></pre>
|
|
|
|
|
|
<h4 id="連接測試資料庫-建立pg-prewarm-extension">連接測試資料庫、建立pg_prewarm extension</h4>
|
|
|
|
|
|
<pre><code>postgres=# \c test ;
|
|
|
You are now connected to database "test" as user "postgres".
|
|
|
test=# CREATE EXTENSION pg_prewarm;
|
|
|
CREATE EXTENSION
|
|
|
test=#
|
|
|
</code></pre>
|
|
|
|
|
|
<h4 id="建立測試資料表-塞入500萬筆資料">建立測試資料表，塞入500萬筆資料</h4>
|
|
|
|
|
|
<pre><code>test=# \timing
|
|
|
Timing is on.
|
|
|
test=# CREATE TABLE test_tbl AS
|
|
|
SELECT floor(random() * (9923123) + 1)::int FROM generate_series(1, 5000000) AS id;
|
|
|
SELECT 5000000
|
|
|
Time: 2940.602 ms (00:02.941)
|
|
|
test=#
|
|
|
</code></pre>
|
|
|
|
|
|
<h4 id="檢查看看剛剛建立的table-用了多少空間">檢查看看剛剛建立的table 用了多少空間</h4>
|
|
|
|
|
|
<p>哎呀，看起來用得不多啊</p>
|
|
|
|
|
|
<pre><code>test=# SELECT pg_size_pretty(pg_relation_size('test_tbl'));
|
|
|
173 MB
|
|
|
</code></pre>
|
|
|
|
|
|
<p><strong>玩大一點，塞個一億筆資料好了</strong></p>
|
|
|
|
|
|
<pre><code>test=# drop table test_tbl;
|
|
|
Time: 0.361 ms
|
|
|
test=# CREATE TABLE test_tbl AS
|
|
|
SELECT floor(random() * (99343) + 1)::int FROM generate_series(1, 100000000) AS id;
|
|
|
SELECT 100000000
|
|
|
Time: 6321.415 ms (00:06.321)
|
|
|
|
|
|
test=# SELECT pg_size_pretty(pg_relation_size('test_tbl'));
|
|
|
pg_size_pretty | 3457 MB
|
|
|
|
|
|
Time: 0.589 ms
|
|
|
test=#
|
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
<p>好，現在資料庫長到3457MB了</p>
|
|
|
|
|
|
<p>先來執行一些初步的取得基本數據</p>
|
|
|
|
|
|
<pre><code>test=# explain (analyze,buffers) select count(*) from test_tbl;
|
|
|
QUERY PLAN
|
|
|
------------------------------------------------------------------------------------------------------------------------
|
|
|
Finalize Aggregate (cost=755978.52..755978.53 rows=1 width=8) (actual time=3331.917..3331.918 rows=1 loops=1)
|
|
|
Buffers: shared hit=160 read=442318
|
|
|
-> Gather (cost=755978.10..755978.51 rows=4 width=8) (actual time=3331.876..3333.674 rows=5 loops=1)
|
|
|
Workers Planned: 4
|
|
|
Workers Launched: 4
|
|
|
Buffers: shared hit=160 read=442318
|
|
|
-> Partial Aggregate (cost=754978.10..754978.11 rows=1 width=8) (actual time=3329.279..3329.280 rows=1 loops=5)
|
|
|
Buffers: shared hit=160 read=442318
|
|
|
-> Parallel Seq Scan on test_tbl (cost=0.00..692478.08 rows=25000008 width=0) (actual time=0.029..1924.601 rows=20000000 loops=5)
|
|
|
Buffers: shared hit=160 read=442318
|
|
|
Planning Time: 0.040 ms
|
|
|
Execution Time: 3333.729 ms
|
|
|
(12 rows)
|
|
|
|
|
|
(END)
|
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
<p>可以看到打中buffer 的部份其實很少，只有 160 ，大部分都是讀進去buffer (442318)</p>
|
|
|
|
|
|
<p>來看看 buffer 的使用狀況</p>
|
|
|
|
|
|
<pre><code>test=# CREATE EXTENSION pg_buffercache;
|
|
|
CREATE EXTENSION
|
|
|
test=# select c.relname,pg_size_pretty(count(*) * 8192) as buffered,
|
|
|
test-# round(100.0 * count(*) / (
|
|
|
test(# select setting from pg_settings
|
|
|
test(# where name='shared_buffers')::integer,1)
|
|
|
test-# as buffer_percent,
|
|
|
test-# round(100.0*count(*)*8192 / pg_table_size(c.oid),1) as percent_of_relation
|
|
|
test-# from pg_class c inner join pg_buffercache b on b.relfilenode = c.relfilenode inner
|
|
|
test-# join pg_database d on ( b.reldatabase =d.oid and d.datname =current_database())
|
|
|
test-# group by c.oid,c.relname order by 3 desc limit 10;
|
|
|
relname | buffered | buffer_percent | percent_of_relation
|
|
|
--------------+------------+----------------+---------------------
|
|
|
test_tbl | 18 MB | 0.3 | 0.5
|
|
|
pg_am | 8192 bytes | 0.0 | 20.0
|
|
|
pg_index | 24 kB | 0.0 | 37.5
|
|
|
pg_amproc | 32 kB | 0.0 | 50.0
|
|
|
pg_cast | 16 kB | 0.0 | 33.3
|
|
|
pg_depend | 64 kB | 0.0 | 13.3
|
|
|
pg_amop | 48 kB | 0.0 | 54.5
|
|
|
pg_namespace | 8192 bytes | 0.0 | 20.0
|
|
|
pg_opclass | 16 kB | 0.0 | 28.6
|
|
|
pg_aggregate | 8192 bytes | 0.0 | 16.7
|
|
|
(10 rows)
|
|
|
|
|
|
Time: 148.719 ms
|
|
|
test=#
|
|
|
</code></pre>
|
|
|
|
|
|
<p>可以看到這個 test_tbl 只有0.5% 被撈到shared_buffers 裡面</p>
|
|
|
|
|
|
<p>接下來就把這個table全部推到shared_buffers 裡面去</p>
|
|
|
|
|
|
<pre><code>test=# select pg_prewarm('test_tbl','buffer');
|
|
|
pg_prewarm
|
|
|
------------
|
|
|
442478
|
|
|
(1 row)
|
|
|
|
|
|
Time: 1938.043 ms (00:01.938)
|
|
|
test=#
|
|
|
</code></pre>
|
|
|
|
|
|
<p>然後再來看一次shared_buffers的使用狀況</p>
|
|
|
|
|
|
<pre><code>test=# select c.relname,pg_size_pretty(count(*) * 8192) as buffered,
|
|
|
round(100.0 * count(*) / (
|
|
|
select setting from pg_settings
|
|
|
where name='shared_buffers')::integer,1)
|
|
|
as buffer_percent,
|
|
|
round(100.0*count(*)*8192 / pg_table_size(c.oid),1) as percent_of_relation
|
|
|
from pg_class c inner join pg_buffercache b on b.relfilenode = c.relfilenode inner
|
|
|
join pg_database d on ( b.reldatabase =d.oid and d.datname =current_database())
|
|
|
group by c.oid,c.relname order by 3 desc limit 10;
|
|
|
relname | buffered | buffer_percent | percent_of_relation
|
|
|
--------------+------------+----------------+---------------------
|
|
|
test_tbl | 3457 MB | 56.3 | 100.0
|
|
|
pg_am | 8192 bytes | 0.0 | 20.0
|
|
|
pg_index | 24 kB | 0.0 | 37.5
|
|
|
pg_amproc | 32 kB | 0.0 | 50.0
|
|
|
pg_cast | 16 kB | 0.0 | 33.3
|
|
|
pg_depend | 64 kB | 0.0 | 13.3
|
|
|
pg_amop | 48 kB | 0.0 | 54.5
|
|
|
pg_namespace | 8192 bytes | 0.0 | 20.0
|
|
|
pg_opclass | 16 kB | 0.0 | 28.6
|
|
|
pg_aggregate | 8192 bytes | 0.0 | 16.7
|
|
|
(10 rows)
|
|
|
|
|
|
Time: 2778.354 ms (00:02.778)
|
|
|
test=#
|
|
|
</code></pre>
|
|
|
|
|
|
<p>OK ，可以看到 test_tbl 已經通通被載入 shared_buffers 中</p>
|
|
|
|
|
|
<p><strong>buffered 表示表格被載入shared_buffers的大小</strong></p>
|
|
|
|
|
|
<p><strong>buffer_percent 表示這個表格佔用多少shared_buffers 的比例</strong></p>
|
|
|
|
|
|
<p><strong>percent_of_relation 表示這個表格有多少比例被載入 shared_buffers</strong></p>
|
|
|
|
|
|
<p>再來跑一次explain看看狀況</p>
|
|
|
|
|
|
<pre><code>test=# explain (analyze,buffers) select count(*) from test_tbl;
|
|
|
Time: 3551.785 ms (00:03.552)
|
|
|
Finalize Aggregate (cost=755978.52..755978.53 rows=1 width=8) (actual time=3427.286..3427.287 rows=1 loops=1)
|
|
|
Buffers: shared hit=442478
|
|
|
-> Gather (cost=755978.10..755978.51 rows=4 width=8) (actual time=3427.215..3551.326 rows=5 loops=1)
|
|
|
Workers Planned: 4
|
|
|
Workers Launched: 4
|
|
|
Buffers: shared hit=442478
|
|
|
-> Partial Aggregate (cost=754978.10..754978.11 rows=1 width=8) (actual time=3423.659..3423.659 rows=1 loops=5)
|
|
|
Buffers: shared hit=442478
|
|
|
-> Parallel Seq Scan on test_tbl (cost=0.00..692478.08 rows=25000008 width=0) (actual time=0.017..1976.744 rows=20000000 loops=5)
|
|
|
Buffers: shared hit=442478
|
|
|
Planning Time: 0.039 ms
|
|
|
Execution Time: 3551.365 ms
|
|
|
(12 rows)
|
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
<p>這邊就可以看到都是從buffer 讀出來所以 hit=442478</p>
|
|
|
|
|
|
<p>看樣子表格還是太小，所以沒有完全發揮？那再來把表格加大！</p>
|
|
|
|
|
|
<p>先重開一次 postgresql 清除buffer</p>
|
|
|
|
|
|
<p>然後重新建立表格</p>
|
|
|
|
|
|
<pre><code>test=# drop table test_tbl;
|
|
|
DROP TABLE
|
|
|
Time: 297.493 ms
|
|
|
test=# CREATE TABLE test_tbl AS
|
|
|
test-# SELECT floor(random() * (993343) + 1)::int FROM generate_series(1, 300000000) AS id;
|
|
|
SELECT 300000000
|
|
|
Time: 290660.607 ms (04:50.661)
|
|
|
test=#
|
|
|
</code></pre>
|
|
|
|
|
|
<p>一樣，看看用了多少容量</p>
|
|
|
|
|
|
<pre><code>test=# SELECT pg_size_pretty(pg_relation_size('test_tbl'));
|
|
|
pg_size_pretty
|
|
|
----------------
|
|
|
10 GB
|
|
|
(1 row)
|
|
|
|
|
|
Time: 0.474 ms
|
|
|
test=#
|
|
|
</code></pre>
|
|
|
|
|
|
<p>哇哈哈，用了10G ，這次還不撐爆你！</p>
|
|
|
|
|
|
<p>跑explain 看看狀況</p>
|
|
|
|
|
|
<pre><code>test=# explain (analyze,buffers) select count(*) from test_tbl;
|
|
|
Time: 22909.065 ms (00:22.909)
|
|
|
|
|
|
QUERY PLAN
|
|
|
---------------------------------------------------------------------------------------------------------------------------
|
|
|
Finalize Aggregate (cost=2265934.72..2265934.73 rows=1 width=8) (actual time=22906.045..22906.045 rows=1 loops=1)
|
|
|
Buffers: shared hit=2080 read=1325354 dirtied=1295425 written=1295265
|
|
|
-> Gather (cost=2265934.30..2265934.71 rows=4 width=8) (actual time=22905.997..22908.522 rows=5 loops=1)
|
|
|
Workers Planned: 4
|
|
|
Workers Launched: 4
|
|
|
Buffers: shared hit=2080 read=1325354 dirtied=1295425 written=1295265
|
|
|
-> Partial Aggregate (cost=2264934.30..2264934.31 rows=1 width=8) (actual time=22903.473..22903.474 rows=1 loops=5)
|
|
|
Buffers: shared hit=2080 read=1325354 dirtied=1295425 written=1295265
|
|
|
-> Parallel Seq Scan on test_tbl (cost=0.00..2077434.24 rows=75000024 width=0) (actual time=0.040..18374.277 rows=60000000 loops=5)
|
|
|
Buffers: shared hit=2080 read=1325354 dirtied=1295425 written=1295265
|
|
|
Planning Time: 0.094 ms
|
|
|
Execution Time: 22908.571 ms
|
|
|
(12 rows)
|
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
<p>看一下現在 shared_buffers 使用狀況</p>
|
|
|
|
|
|
<p>可以看到這個 test_tbl 幾乎沒被放入 shared_buffers 中</p>
|
|
|
|
|
|
<pre><code>test=# select c.relname,pg_size_pretty(count(*) * 8192) as buffered,
|
|
|
round(100.0 * count(*) / (
|
|
|
select setting from pg_settings
|
|
|
where name='shared_buffers')::integer,1)
|
|
|
as buffer_percent,
|
|
|
round(100.0*count(*)*8192 / pg_table_size(c.oid),1) as percent_of_relation
|
|
|
from pg_class c inner join pg_buffercache b on b.relfilenode = c.relfilenode inner
|
|
|
join pg_database d on ( b.reldatabase =d.oid and d.datname =current_database())
|
|
|
group by c.oid,c.relname order by 3 desc limit 10;
|
|
|
relname | buffered | buffer_percent | percent_of_relation
|
|
|
--------------+------------+----------------+---------------------
|
|
|
test_tbl | 18 MB | 0.3 | 0.2
|
|
|
pg_am | 8192 bytes | 0.0 | 20.0
|
|
|
pg_index | 24 kB | 0.0 | 37.5
|
|
|
pg_amproc | 32 kB | 0.0 | 50.0
|
|
|
pg_cast | 16 kB | 0.0 | 33.3
|
|
|
pg_depend | 64 kB | 0.0 | 13.3
|
|
|
pg_amop | 48 kB | 0.0 | 54.5
|
|
|
pg_namespace | 8192 bytes | 0.0 | 20.0
|
|
|
pg_opclass | 16 kB | 0.0 | 28.6
|
|
|
pg_aggregate | 8192 bytes | 0.0 | 16.7
|
|
|
(10 rows)
|
|
|
|
|
|
Time: 163.936 ms
|
|
|
test=#
|
|
|
</code></pre>
|
|
|
|
|
|
<p>強制把test_tbl 全部塞進 shared_buffers</p>
|
|
|
|
|
|
<pre><code>test=# select pg_prewarm('test_tbl','buffer');
|
|
|
pg_prewarm
|
|
|
------------
|
|
|
1327434
|
|
|
(1 row)
|
|
|
|
|
|
Time: 7472.805 ms (00:07.473)
|
|
|
test=#
|
|
|
</code></pre>
|
|
|
|
|
|
<p>確認一下test_tbl 有沒有被整個塞進去</p>
|
|
|
|
|
|
<pre><code>test=# select c.relname,pg_size_pretty(count(*) * 8192) as buffered,
|
|
|
round(100.0 * count(*) / (
|
|
|
select setting from pg_settings
|
|
|
where name='shared_buffers')::integer,1)
|
|
|
as buffer_percent,
|
|
|
round(100.0*count(*)*8192 / pg_table_size(c.oid),1) as percent_of_relation
|
|
|
from pg_class c inner join pg_buffercache b on b.relfilenode = c.relfilenode inner
|
|
|
join pg_database d on ( b.reldatabase =d.oid and d.datname =current_database())
|
|
|
group by c.oid,c.relname order by 3 desc limit 10;
|
|
|
relname | buffered | buffer_percent | percent_of_relation
|
|
|
--------------+------------+----------------+---------------------
|
|
|
test_tbl | 6142 MB | 100.0 | 59.2
|
|
|
pg_am | 8192 bytes | 0.0 | 20.0
|
|
|
pg_index | 24 kB | 0.0 | 37.5
|
|
|
pg_amproc | 32 kB | 0.0 | 50.0
|
|
|
pg_cast | 16 kB | 0.0 | 33.3
|
|
|
pg_depend | 24 kB | 0.0 | 5.0
|
|
|
pg_amop | 40 kB | 0.0 | 45.5
|
|
|
pg_namespace | 8192 bytes | 0.0 | 20.0
|
|
|
pg_opclass | 16 kB | 0.0 | 28.6
|
|
|
pg_aggregate | 8192 bytes | 0.0 | 16.7
|
|
|
(10 rows)
|
|
|
|
|
|
Time: 4985.366 ms (00:04.985)
|
|
|
test=#
|
|
|
</code></pre>
|
|
|
|
|
|
<p>GOOD ！ let’s do explain again !</p>
|
|
|
|
|
|
<pre><code>test=# explain (analyze,buffers) select count(*) from test_tbl;
|
|
|
Time: 11451.188 ms (00:11.451)
|
|
|
QUERY PLAN
|
|
|
--------------------------------------------------------------------------------------------------------------------
|
|
|
Finalize Aggregate (cost=2265934.72..2265934.73 rows=1 width=8) (actual time=11231.664..11231.664 rows=1 loops=1)
|
|
|
Buffers: shared hit=785963 read=541471
|
|
|
-> Gather (cost=2265934.30..2265934.71 rows=4 width=8) (actual time=11231.606..11450.719 rows=5 loops=1)
|
|
|
Workers Planned: 4
|
|
|
Workers Launched: 4
|
|
|
Buffers: shared hit=785963 read=541471
|
|
|
-> Partial Aggregate (cost=2264934.30..2264934.31 rows=1 width=8) (actual time=11228.829..11228.830 rows=1 loops=5)
|
|
|
Buffers: shared hit=785963 read=541471
|
|
|
-> Parallel Seq Scan on test_tbl (cost=0.00..2077434.24 rows=75000024 width=0) (actual time=0.037..6414.711 rows=60000000 loops=5)
|
|
|
Buffers: shared hit=785963 read=541471
|
|
|
Planning Time: 0.039 ms
|
|
|
Execution Time: 11450.781 ms
|
|
|
(12 rows)
|
|
|
</code></pre>
|
|
|
|
|
|
<p>確認一下，果然大部分都打到cache 了，但是因為shared_buffers 不夠大，所以還會從磁碟讀取一部分</p>
|
|
|
|
|
|
<p>而且時間也比之前還沒塞進shared_buffers 的時候要快了不少</p>
|
|
|
|
|
|
<p>22908.571 –> 11450.781 ms</p>
|
|
|
|
|
|
<hr />
|
|
|
|
|
|
<p>從這次的測試看來，我想如果有足夠大的記憶體，能夠把資料表都塞入shared_buffers 中</p>
|
|
|
|
|
|
<p>應該可以帶來不錯的效能增幅！</p>
|
|
|
</article>
|
|
|
</div>
|
|
|
|
|
|
<div class='entry-meta-bottom'>
|
|
|
|
|
|
|
|
|
<div class="entry-categories"><p><span>Categories</span>
|
|
|
|
|
|
<a href="/categories/%E7%AD%86%E8%A8%98" title="View all posts in 筆記">筆記</a>
|
|
|
</p>
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
<div class="entry-tags"><p><span>Tags</span>
|
|
|
|
|
|
<a href="/tags/postgresql" title="View all posts tagged postgresql">postgresql</a>
|
|
|
|
|
|
|
|
|
</p></div> </div>
|
|
|
|
|
|
|
|
|
<div class="author-meta">
|
|
|
|
|
|
<div class="author">
|
|
|
|
|
|
<img alt='Eric Chang' src="https://www.gravatar.com/avatar/23f8ed94e007297499ac8df1641b3ff5?s=100&d=identicon" class='avatar avatar-72 photo' height='72' width='72'>
|
|
|
|
|
|
<span>
|
|
|
Written by:<a href="https://github.com/changchichung" title="Posts by Eric Chang" rel="author">Eric Chang</a> </span>
|
|
|
</div>
|
|
|
<div class="bio">
|
|
|
|
|
|
|
|
|
<p>塵世裡一個迷途小書僮</p>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<a class="facebook" target="_blank"
|
|
|
href="full%20Social%20profile%20url%20in%20facebook">
|
|
|
<i class="fab fa-facebook-f"
|
|
|
title="facebook icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
<a class="googleplus" target="_blank"
|
|
|
href="full%20profile%20url%20in%20googleplus">
|
|
|
<i class="fab fa-google-plus-g"
|
|
|
title="googleplus icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<a class="twitter" target="_blank"
|
|
|
href="chang0206">
|
|
|
<i class="fab fa-twitter-square"
|
|
|
title="twitter icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
<a class="linkedin" target="_blank"
|
|
|
href="full%20profile%20url%20in%20linkedin">
|
|
|
<i class="fab fa-linkedin"
|
|
|
title="linkedin icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
<a class="email" target="_blank"
|
|
|
href="mailto:mc@hotshraingmy.info">
|
|
|
<i class="fas fa-envelope"
|
|
|
title="email icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
<a class="instagram" target="_blank"
|
|
|
href="chang0206">
|
|
|
<i class="fab fa-instagram"
|
|
|
title="instagram icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
<a class="stackoverflow" target="_blank"
|
|
|
href="full%20profile%20url%20in%20stackoverflow">
|
|
|
<i class="fab fa-stack-overflow"
|
|
|
title="stackoverflow icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
<a class="github" target="_blank"
|
|
|
href="changchichung">
|
|
|
<i class="fab fa-github"
|
|
|
title="github icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<a class="pinterest" target="_blank"
|
|
|
href="full%20profile%20url%20in%20pinterest">
|
|
|
<i class="fab fa-pinterest"
|
|
|
title="pinterest icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
</div>
|
|
|
|
|
|
<section id="comments" class="comments">
|
|
|
<div id="disqus_thread"></div>
|
|
|
<script type="application/javascript">
|
|
|
var disqus_config = function () {
|
|
|
|
|
|
|
|
|
|
|
|
};
|
|
|
(function() {
|
|
|
if (["localhost", "127.0.0.1"].indexOf(window.location.hostname) != -1) {
|
|
|
document.getElementById('disqus_thread').innerHTML = 'Disqus comments not available by default when the website is previewed locally.';
|
|
|
return;
|
|
|
}
|
|
|
var d = document, s = d.createElement('script'); s.async = true;
|
|
|
s.src = '//' + "h-cowbay-org-1" + '.disqus.com/embed.js';
|
|
|
s.setAttribute('data-timestamp', +new Date());
|
|
|
(d.head || d.body).appendChild(s);
|
|
|
})();
|
|
|
</script>
|
|
|
<noscript>Please enable JavaScript to view the <a href="https://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
|
|
|
<a href="https://disqus.com" class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</section>
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
|
|
|
<footer id="site-footer" class="site-footer" role="contentinfo">
|
|
|
<h1>
|
|
|
|
|
|
<a href=""> MC部落 </a>
|
|
|
|
|
|
</h1>
|
|
|
|
|
|
|
|
|
<p class="site-description">What’s the Worst That Could Happen?</p>
|
|
|
|
|
|
|
|
|
<div id="menu-footer" class="menu-container menu-footer" role="navigation">
|
|
|
<div class="menu">
|
|
|
|
|
|
<ul id="menu-footer-items" class="menu-footer-items">
|
|
|
|
|
|
</ul>
|
|
|
|
|
|
</div> </div>
|
|
|
|
|
|
<ul class="social-media-icons">
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a class="facebook" target="_blank"
|
|
|
href="full%20Social%20profile%20url%20in%20facebook" >
|
|
|
<i class="fab fa-facebook-f" title="facebook"></i>
|
|
|
<span class="screen-reader-text">facebook</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a class="googleplus" target="_blank"
|
|
|
href="full%20profile%20url%20in%20googleplus" >
|
|
|
<i class="fab fa-google-plus-g" title="googleplus"></i>
|
|
|
<span class="screen-reader-text">googleplus</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="chang0206" class="twitter" target="_blank">
|
|
|
<i class="fab fa-twitter-square" title="twitter"></i>
|
|
|
<span class="screen-reader-text">twitter</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="chang0206" class="instagram" target="_blank">
|
|
|
<i class="fab fa-instagram" title="instagram"></i>
|
|
|
<span class="screen-reader-text">instagram</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="mailto:mc@hotshraingmy.info" class="email">
|
|
|
<i class="fas fa-envelope" title="email"></i>
|
|
|
<span class="screen-reader-text">email</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20linkedin" class="linkedin" target="_blank">
|
|
|
<i class="fab fa-linkedin-in" title="linkedin"></i>
|
|
|
<span class="screen-reader-text">linkedin</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20stackoverflow" class="stackoverflow" target="_blank">
|
|
|
<i class="fab fa-stack-overflow" title="stackoverflow"></i>
|
|
|
<span class="screen-reader-text">stackoverflow</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="changchichung" class="github" target="_blank">
|
|
|
<i class="fab fa-github" title="github"></i>
|
|
|
<span class="screen-reader-text">github</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20pinterest" class="pinterest" target="_blank">
|
|
|
<i class="fab fa-pinterest" title="pinterest"></i>
|
|
|
<span class="screen-reader-text">pinterest</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="https://h.cowbay.org/index.xml" data-animate-hover="pulse" class="rss" target="_blank">
|
|
|
<i class="fas fa-rss" title="rss"></i>
|
|
|
<span class="screen-reader-text">rss</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
</ul> <div class="design-credit">
|
|
|
|
|
|
<p>© 2018 Göran Svensson</p>
|
|
|
|
|
|
<p>Nederburg Hugo Theme by <a href="https://appernetic.io">Appernetic</a>.</p>
|
|
|
|
|
|
<p>A port of Tracks by Compete Themes.</p>
|
|
|
|
|
|
</div>
|
|
|
</footer>
|
|
|
|
|
|
</div>
|
|
|
<script src="https://h.cowbay.org/js/jquery.min.js"></script>
|
|
|
<script src="https://h.cowbay.org/js/jquerymigrate.js"></script>
|
|
|
<script src="https://h.cowbay.org/js/production.min.js?v=1626744134"></script>
|
|
|
|
|
|
</body>
|
|
|
</html>
|
