add auto fetch ssl certs
This commit is contained in:
@@ -13,7 +13,7 @@
|
||||
"articleSection" : "post",
|
||||
"name" : "[筆記] 測試 postgresql 的pg_prewarm 對效能的影響 \/ test pg_prewarm in postgresql 11",
|
||||
"headline" : "[筆記] 測試 postgresql 的pg_prewarm 對效能的影響 \/ test pg_prewarm in postgresql 11",
|
||||
"description" : "\x3cp\x3e老闆提到想要把新系統的 postgresql 資料庫都撈到記憶體裡面\x3c\/p\x3e\n\n\x3cp\x3e但是否決了我提出的ramdisk 作法(因為當機的話，資料就沒了)\x3c\/p\x3e\n\n\x3cp\x3e在找資料的時候，發現了這個postgresql 的 pg_prewarm extension\x3c\/p\x3e\n\n\x3cp\x3e好像有點意思？就來測試看看吧！\x3c\/p\x3e\n\n\x3cp\x3e只是目前還不知道該怎麼解讀測試的數據就是了\x26hellip;\x3c\/p\x3e\n\n\x3cp\x3e幹！林北真的不是 DBA 啦 =.=\x3c\/p\x3e",
|
||||
"description" : "\x3cp\x3e老闆提到想要把新系統的 postgresql 資料庫都撈到記憶體裡面\x3c\/p\x3e\n\x3cp\x3e但是否決了我提出的ramdisk 作法(因為當機的話，資料就沒了)\x3c\/p\x3e\n\x3cp\x3e在找資料的時候，發現了這個postgresql 的 pg_prewarm extension\x3c\/p\x3e\n\x3cp\x3e好像有點意思？就來測試看看吧！\x3c\/p\x3e\n\x3cp\x3e只是目前還不知道該怎麼解讀測試的數據就是了\x26hellip;\x3c\/p\x3e\n\x3cp\x3e幹！林北真的不是 DBA 啦 =.=\x3c\/p\x3e",
|
||||
"inLanguage" : "en",
|
||||
"author" : "Eric Chang",
|
||||
"creator" : "Eric Chang",
|
||||
@@ -45,9 +45,9 @@
|
||||
|
||||
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.3.1/css/all.css" integrity="sha384-mzrmE5qonljUremFsqc01SB46JvROS7bZs3IO2EmfFsd15uHvIt+Y8vEf7N7fWAU" crossorigin="anonymous">
|
||||
|
||||
<link href="https://h.cowbay.org/css/style.css?v=1626744134" rel="stylesheet" id="theme-stylesheet" type='text/css' media='all'>
|
||||
<link href="https://h.cowbay.org/css/style.css?v=1629951055" rel="stylesheet" id="theme-stylesheet" type='text/css' media='all'>
|
||||
|
||||
<link href="https://h.cowbay.org/css/custom.css?v=1626744134" rel="stylesheet" type='text/css' media='all'>
|
||||
<link href="https://h.cowbay.org/css/custom.css?v=1629951055" rel="stylesheet" type='text/css' media='all'>
|
||||
<link rel="shortcut icon" href="https://h.cowbay.org/img/favicon.ico" type="image/x-icon">
|
||||
<link rel="icon" href="https://h.cowbay.org/img/favicon.ico" type="image/x-icon">
|
||||
|
||||
@@ -81,10 +81,6 @@ if (!doNotTrack) {
|
||||
|
||||
<ul id="menu-secondary-items" class="menu-secondary-items">
|
||||
|
||||
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
||||
<a href="/categories/"></a>
|
||||
</li>
|
||||
|
||||
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
||||
<a href="/categories/ansible">ansible</a>
|
||||
</li>
|
||||
@@ -309,25 +305,15 @@ if (!doNotTrack) {
|
||||
<div class="entry-content">
|
||||
<article>
|
||||
<p>老闆提到想要把新系統的 postgresql 資料庫都撈到記憶體裡面</p>
|
||||
|
||||
<p>但是否決了我提出的ramdisk 作法(因為當機的話，資料就沒了)</p>
|
||||
|
||||
<p>在找資料的時候，發現了這個postgresql 的 pg_prewarm extension</p>
|
||||
|
||||
<p>好像有點意思？就來測試看看吧！</p>
|
||||
|
||||
<p>只是目前還不知道該怎麼解讀測試的數據就是了…</p>
|
||||
|
||||
<p>幹！林北真的不是 DBA 啦 =.=</p>
|
||||
|
||||
<p>安裝系統、postgresql 資料庫什麼的就不提了，那不是這次的重點</p>
|
||||
|
||||
<h4 id="修改-postgresql-conf">修改 postgresql.conf</h4>
|
||||
|
||||
<h4 id="修改-postgresqlconf">修改 postgresql.conf</h4>
|
||||
<p>編輯postgresql.conf，開啟平行處理以及設定可用記憶體容量</p>
|
||||
|
||||
<p>這台測試機的環境是一台三代i7 , 24G RAM , 240G SSD，安裝debian 10(buster)</p>
|
||||
|
||||
<pre><code># load libiriaes
|
||||
# 其實這次不會用到pg_stat_statements ，不過出於習慣，還是加入開機自動載入吧
|
||||
shared_preload_libraries = 'pg_stat_statements'
|
||||
@@ -351,30 +337,19 @@ max_wal_size = 2GB
|
||||
max_worker_processes = 8
|
||||
max_parallel_workers_per_gather = 4
|
||||
max_parallel_workers = 8
|
||||
</code></pre>
|
||||
|
||||
<p>重新啟動postgresql ，準備開始測試囉！</p>
|
||||
|
||||
</code></pre><p>重新啟動postgresql ，準備開始測試囉！</p>
|
||||
<p>轉換成 postgres 身份後，進入 psql</p>
|
||||
|
||||
<h4 id="建立測試資料庫">建立測試資料庫</h4>
|
||||
|
||||
<pre><code>postgres=# create database test;
|
||||
CREATE DATABASE
|
||||
postgres=#
|
||||
</code></pre>
|
||||
|
||||
<h4 id="連接測試資料庫-建立pg-prewarm-extension">連接測試資料庫、建立pg_prewarm extension</h4>
|
||||
|
||||
</code></pre><h4 id="連接測試資料庫建立pg_prewarm-extension">連接測試資料庫、建立pg_prewarm extension</h4>
|
||||
<pre><code>postgres=# \c test ;
|
||||
You are now connected to database "test" as user "postgres".
|
||||
test=# CREATE EXTENSION pg_prewarm;
|
||||
CREATE EXTENSION
|
||||
test=#
|
||||
</code></pre>
|
||||
|
||||
<h4 id="建立測試資料表-塞入500萬筆資料">建立測試資料表，塞入500萬筆資料</h4>
|
||||
|
||||
</code></pre><h4 id="建立測試資料表塞入500萬筆資料">建立測試資料表，塞入500萬筆資料</h4>
|
||||
<pre><code>test=# \timing
|
||||
Timing is on.
|
||||
test=# CREATE TABLE test_tbl AS
|
||||
@@ -382,18 +357,11 @@ SELECT floor(random() * (9923123) + 1)::int FROM generate_series(1, 5000000) AS
|
||||
SELECT 5000000
|
||||
Time: 2940.602 ms (00:02.941)
|
||||
test=#
|
||||
</code></pre>
|
||||
|
||||
<h4 id="檢查看看剛剛建立的table-用了多少空間">檢查看看剛剛建立的table 用了多少空間</h4>
|
||||
|
||||
</code></pre><h4 id="檢查看看剛剛建立的table-用了多少空間">檢查看看剛剛建立的table 用了多少空間</h4>
|
||||
<p>哎呀，看起來用得不多啊</p>
|
||||
|
||||
<pre><code>test=# SELECT pg_size_pretty(pg_relation_size('test_tbl'));
|
||||
173 MB
|
||||
</code></pre>
|
||||
|
||||
<p><strong>玩大一點，塞個一億筆資料好了</strong></p>
|
||||
|
||||
</code></pre><p><strong>玩大一點，塞個一億筆資料好了</strong></p>
|
||||
<pre><code>test=# drop table test_tbl;
|
||||
Time: 0.361 ms
|
||||
test=# CREATE TABLE test_tbl AS
|
||||
@@ -407,12 +375,8 @@ pg_size_pretty | 3457 MB
|
||||
Time: 0.589 ms
|
||||
test=#
|
||||
|
||||
</code></pre>
|
||||
|
||||
<p>好，現在資料庫長到3457MB了</p>
|
||||
|
||||
</code></pre><p>好，現在資料庫長到3457MB了</p>
|
||||
<p>先來執行一些初步的取得基本數據</p>
|
||||
|
||||
<pre><code>test=# explain (analyze,buffers) select count(*) from test_tbl;
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
@@ -432,12 +396,8 @@ test=#
|
||||
|
||||
(END)
|
||||
|
||||
</code></pre>
|
||||
|
||||
<p>可以看到打中buffer 的部份其實很少，只有 160 ，大部分都是讀進去buffer (442318)</p>
|
||||
|
||||
</code></pre><p>可以看到打中buffer 的部份其實很少，只有 160 ，大部分都是讀進去buffer (442318)</p>
|
||||
<p>來看看 buffer 的使用狀況</p>
|
||||
|
||||
<pre><code>test=# CREATE EXTENSION pg_buffercache;
|
||||
CREATE EXTENSION
|
||||
test=# select c.relname,pg_size_pretty(count(*) * 8192) as buffered,
|
||||
@@ -465,12 +425,8 @@ test-# group by c.oid,c.relname order by 3 desc limit 10;
|
||||
|
||||
Time: 148.719 ms
|
||||
test=#
|
||||
</code></pre>
|
||||
|
||||
<p>可以看到這個 test_tbl 只有0.5% 被撈到shared_buffers 裡面</p>
|
||||
|
||||
</code></pre><p>可以看到這個 test_tbl 只有0.5% 被撈到shared_buffers 裡面</p>
|
||||
<p>接下來就把這個table全部推到shared_buffers 裡面去</p>
|
||||
|
||||
<pre><code>test=# select pg_prewarm('test_tbl','buffer');
|
||||
pg_prewarm
|
||||
------------
|
||||
@@ -479,10 +435,7 @@ test=#
|
||||
|
||||
Time: 1938.043 ms (00:01.938)
|
||||
test=#
|
||||
</code></pre>
|
||||
|
||||
<p>然後再來看一次shared_buffers的使用狀況</p>
|
||||
|
||||
</code></pre><p>然後再來看一次shared_buffers的使用狀況</p>
|
||||
<pre><code>test=# select c.relname,pg_size_pretty(count(*) * 8192) as buffered,
|
||||
round(100.0 * count(*) / (
|
||||
select setting from pg_settings
|
||||
@@ -508,18 +461,11 @@ group by c.oid,c.relname order by 3 desc limit 10;
|
||||
|
||||
Time: 2778.354 ms (00:02.778)
|
||||
test=#
|
||||
</code></pre>
|
||||
|
||||
<p>OK ，可以看到 test_tbl 已經通通被載入 shared_buffers 中</p>
|
||||
|
||||
</code></pre><p>OK ，可以看到 test_tbl 已經通通被載入 shared_buffers 中</p>
|
||||
<p><strong>buffered 表示表格被載入shared_buffers的大小</strong></p>
|
||||
|
||||
<p><strong>buffer_percent 表示這個表格佔用多少shared_buffers 的比例</strong></p>
|
||||
|
||||
<p><strong>percent_of_relation 表示這個表格有多少比例被載入 shared_buffers</strong></p>
|
||||
|
||||
<p>再來跑一次explain看看狀況</p>
|
||||
|
||||
<pre><code>test=# explain (analyze,buffers) select count(*) from test_tbl;
|
||||
Time: 3551.785 ms (00:03.552)
|
||||
Finalize Aggregate (cost=755978.52..755978.53 rows=1 width=8) (actual time=3427.286..3427.287 rows=1 loops=1)
|
||||
@@ -536,16 +482,10 @@ Time: 3551.785 ms (00:03.552)
|
||||
Execution Time: 3551.365 ms
|
||||
(12 rows)
|
||||
|
||||
</code></pre>
|
||||
|
||||
<p>這邊就可以看到都是從buffer 讀出來所以 hit=442478</p>
|
||||
|
||||
</code></pre><p>這邊就可以看到都是從buffer 讀出來所以 hit=442478</p>
|
||||
<p>看樣子表格還是太小，所以沒有完全發揮？那再來把表格加大！</p>
|
||||
|
||||
<p>先重開一次 postgresql 清除buffer</p>
|
||||
|
||||
<p>然後重新建立表格</p>
|
||||
|
||||
<pre><code>test=# drop table test_tbl;
|
||||
DROP TABLE
|
||||
Time: 297.493 ms
|
||||
@@ -554,10 +494,7 @@ test-# SELECT floor(random() * (993343) + 1)::int FROM generate_series(1, 300000
|
||||
SELECT 300000000
|
||||
Time: 290660.607 ms (04:50.661)
|
||||
test=#
|
||||
</code></pre>
|
||||
|
||||
<p>一樣，看看用了多少容量</p>
|
||||
|
||||
</code></pre><p>一樣，看看用了多少容量</p>
|
||||
<pre><code>test=# SELECT pg_size_pretty(pg_relation_size('test_tbl'));
|
||||
pg_size_pretty
|
||||
----------------
|
||||
@@ -566,12 +503,8 @@ test=#
|
||||
|
||||
Time: 0.474 ms
|
||||
test=#
|
||||
</code></pre>
|
||||
|
||||
<p>哇哈哈，用了10G ，這次還不撐爆你！</p>
|
||||
|
||||
</code></pre><p>哇哈哈，用了10G ，這次還不撐爆你！</p>
|
||||
<p>跑explain 看看狀況</p>
|
||||
|
||||
<pre><code>test=# explain (analyze,buffers) select count(*) from test_tbl;
|
||||
Time: 22909.065 ms (00:22.909)
|
||||
|
||||
@@ -591,12 +524,8 @@ Time: 22909.065 ms (00:22.909)
|
||||
Execution Time: 22908.571 ms
|
||||
(12 rows)
|
||||
|
||||
</code></pre>
|
||||
|
||||
<p>看一下現在 shared_buffers 使用狀況</p>
|
||||
|
||||
</code></pre><p>看一下現在 shared_buffers 使用狀況</p>
|
||||
<p>可以看到這個 test_tbl 幾乎沒被放入 shared_buffers 中</p>
|
||||
|
||||
<pre><code>test=# select c.relname,pg_size_pretty(count(*) * 8192) as buffered,
|
||||
round(100.0 * count(*) / (
|
||||
select setting from pg_settings
|
||||
@@ -622,10 +551,7 @@ group by c.oid,c.relname order by 3 desc limit 10;
|
||||
|
||||
Time: 163.936 ms
|
||||
test=#
|
||||
</code></pre>
|
||||
|
||||
<p>強制把test_tbl 全部塞進 shared_buffers</p>
|
||||
|
||||
</code></pre><p>強制把test_tbl 全部塞進 shared_buffers</p>
|
||||
<pre><code>test=# select pg_prewarm('test_tbl','buffer');
|
||||
pg_prewarm
|
||||
------------
|
||||
@@ -634,10 +560,7 @@ test=#
|
||||
|
||||
Time: 7472.805 ms (00:07.473)
|
||||
test=#
|
||||
</code></pre>
|
||||
|
||||
<p>確認一下test_tbl 有沒有被整個塞進去</p>
|
||||
|
||||
</code></pre><p>確認一下test_tbl 有沒有被整個塞進去</p>
|
||||
<pre><code>test=# select c.relname,pg_size_pretty(count(*) * 8192) as buffered,
|
||||
round(100.0 * count(*) / (
|
||||
select setting from pg_settings
|
||||
@@ -663,10 +586,7 @@ group by c.oid,c.relname order by 3 desc limit 10;
|
||||
|
||||
Time: 4985.366 ms (00:04.985)
|
||||
test=#
|
||||
</code></pre>
|
||||
|
||||
<p>GOOD ！ let’s do explain again !</p>
|
||||
|
||||
</code></pre><p>GOOD ！ let’s do explain again !</p>
|
||||
<pre><code>test=# explain (analyze,buffers) select count(*) from test_tbl;
|
||||
Time: 11451.188 ms (00:11.451)
|
||||
QUERY PLAN
|
||||
@@ -684,18 +604,11 @@ Time: 11451.188 ms (00:11.451)
|
||||
Planning Time: 0.039 ms
|
||||
Execution Time: 11450.781 ms
|
||||
(12 rows)
|
||||
</code></pre>
|
||||
|
||||
<p>確認一下，果然大部分都打到cache 了，但是因為shared_buffers 不夠大，所以還會從磁碟讀取一部分</p>
|
||||
|
||||
</code></pre><p>確認一下，果然大部分都打到cache 了，但是因為shared_buffers 不夠大，所以還會從磁碟讀取一部分</p>
|
||||
<p>而且時間也比之前還沒塞進shared_buffers 的時候要快了不少</p>
|
||||
|
||||
<p>22908.571 –> 11450.781 ms</p>
|
||||
|
||||
<hr />
|
||||
|
||||
<hr>
|
||||
<p>從這次的測試看來，我想如果有足夠大的記憶體，能夠把資料表都塞入shared_buffers 中</p>
|
||||
|
||||
<p>應該可以帶來不錯的效能增幅！</p>
|
||||
</article>
|
||||
</div>
|
||||
@@ -971,7 +884,7 @@ title="pinterest icon"></i>
|
||||
|
||||
</ul> <div class="design-credit">
|
||||
|
||||
<p>© 2018 Göran Svensson</p>
|
||||
<p>© 2018 Göran Svensson</p>
|
||||
|
||||
<p>Nederburg Hugo Theme by <a href="https://appernetic.io">Appernetic</a>.</p>
|
||||
|
||||
@@ -983,7 +896,7 @@ title="pinterest icon"></i>
|
||||
</div>
|
||||
<script src="https://h.cowbay.org/js/jquery.min.js"></script>
|
||||
<script src="https://h.cowbay.org/js/jquerymigrate.js"></script>
|
||||
<script src="https://h.cowbay.org/js/production.min.js?v=1626744134"></script>
|
||||
<script src="https://h.cowbay.org/js/production.min.js?v=1629951055"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
Reference in New Issue
Block a user