|
|
<!DOCTYPE html>
|
|
|
<html lang="en-us">
|
|
|
<head><meta charset="utf-8">
|
|
|
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
|
|
<script type="application/ld+json">
|
|
|
{
|
|
|
"@context" : "http://schema.org",
|
|
|
"@type" : "BlogPosting",
|
|
|
"mainEntityOfPage": {
|
|
|
"@type": "WebPage",
|
|
|
"@id": "https:\/\/h.cowbay.org"
|
|
|
},
|
|
|
"articleSection" : "post",
|
|
|
"name" : "[筆記] 用 proxmox \x26 Mellanox SFP 網卡土炮 10G LAB ",
|
|
|
"headline" : "[筆記] 用 proxmox \x26 Mellanox SFP 網卡土炮 10G LAB ",
|
|
|
"description" : "\x3cp\x3e想做一個 10G 的 LAB 環境出來已經很久了。\x3c\/p\x3e\n\n\x3cp\x3e只是礙於10G RJ45的卡太貴了，然後光纖的種類又太複雜\x3c\/p\x3e\n\n\x3cp\x3e如果直接在淘寶購買，很怕會買錯(什麼LC\/FC LC\/LC 多模單模 單芯雙芯 SFP\/SFP\x2b 又是什麼光模塊的一大堆規格)\x3c\/p\x3e\n\n\x3cp\x3e所以一直沒有付諸行動。\x3c\/p\x3e\n\n\x3cp\x3e硬體的工作很久沒碰了，剛好在蝦皮看到有個賣家在賣 mellanox 的X2網卡，以在台灣的價格來說，算很便宜的 (550)\x3c\/p\x3e\n\n\x3cp\x3e聊了一下，跟他請教了關於線材、光纖模塊的問題，回答也都很快很到位\x3c\/p\x3e\n\n\x3cp\x3e就直接下訂了兩張網卡、兩個光纖模塊、一條LC\/LC 光纖線\x3c\/p\x3e\n\n\x3cp\x3e就是到貨有點久，等了兩個禮拜左右，一直到昨天東西才寄到\x3c\/p\x3e\n\n\x3cp\x3e今天就花了點時間測試一下\x3c\/p\x3e",
|
|
|
"inLanguage" : "en",
|
|
|
"author" : "Eric Chang",
|
|
|
"creator" : "Eric Chang",
|
|
|
"publisher": "Eric Chang",
|
|
|
"accountablePerson" : "Eric Chang",
|
|
|
"copyrightHolder" : "Eric Chang",
|
|
|
"copyrightYear" : "2018",
|
|
|
"datePublished": "2018-11-30 16:05:14 \x2b0800 CST",
|
|
|
"dateModified" : "2018-11-30 16:05:14 \x2b0800 CST",
|
|
|
"url" : "https:\/\/h.cowbay.org\/post\/10g-lab-using-proxmox-and-mellanox\/",
|
|
|
"wordCount" : "583",
|
|
|
"image" : "https://h.cowbay.orghttps://h.cowbay.org/images/post-default-03.jpg"",
|
|
|
"keywords" : [ ""10G"",""筆記"",""mellanox"","Blog" ]
|
|
|
}
|
|
|
</script>
|
|
|
|
|
|
|
|
|
<title>[筆記] 用 proxmox & Mellanox SFP 網卡土炮 10G LAB </title>
|
|
|
|
|
|
|
|
|
<meta name="description" content="some articles about job,food,passion sisters" />
|
|
|
|
|
|
|
|
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
|
<meta name="robots" content="all,follow">
|
|
|
<meta name="googlebot" content="index,follow,snippet,archive">
|
|
|
<link rel="stylesheet" id="ct-tracks-google-fonts-css" href="https://fonts.googleapis.com/css?family=Raleway%3A400%2C700&subset=latin%2Clatin-ext&ver=4.7.2" type="text/css" media="all">
|
|
|
|
|
|
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.3.1/css/all.css" integrity="sha384-mzrmE5qonljUremFsqc01SB46JvROS7bZs3IO2EmfFsd15uHvIt+Y8vEf7N7fWAU" crossorigin="anonymous">
|
|
|
|
|
|
<link href="https://h.cowbay.org/css/style.css?v=1577410044" rel="stylesheet" id="theme-stylesheet" type='text/css' media='all'>
|
|
|
|
|
|
<link href="https://h.cowbay.org/css/custom.css?v=1577410044" rel="stylesheet" type='text/css' media='all'>
|
|
|
<link rel="shortcut icon" href="https://h.cowbay.org/img/favicon.ico" type="image/x-icon">
|
|
|
<link rel="icon" href="https://h.cowbay.org/img/favicon.ico" type="image/x-icon">
|
|
|
|
|
|
|
|
|
<script type="application/javascript">
|
|
|
var doNotTrack = false;
|
|
|
if (!doNotTrack) {
|
|
|
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
|
|
|
ga('create', 'UA-138954876-1', 'auto');
|
|
|
|
|
|
ga('send', 'pageview');
|
|
|
}
|
|
|
</script>
|
|
|
<script async src='https://www.google-analytics.com/analytics.js'></script>
|
|
|
|
|
|
</head>
|
|
|
|
|
|
|
|
|
<body class="post-template-default single single-post single-format-standard ct-body singular singular-post not-front standard">
|
|
|
|
|
|
<div id="overflow-container" class="overflow-container">
|
|
|
<a class="skip-content" href="#main">Skip to content</a>
|
|
|
<header id="site-header" class="site-header" role="banner">
|
|
|
<div class='top-navigation'>
|
|
|
<div class='container'>
|
|
|
|
|
|
<div id="menu-secondary" class="menu-container menu-secondary" role="navigation">
|
|
|
<button id="toggle-secondary-navigation" class="toggle-secondary-navigation"><i class="fas fa-plus"></i></button>
|
|
|
|
|
|
<div class="menu">
|
|
|
|
|
|
<ul id="menu-secondary-items" class="menu-secondary-items">
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/"></a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/ansible">ansible</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/linux">linux</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/proxmox">proxmox</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/ps">ps</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/%E7%A2%8E%E5%BF%B5">碎念</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/%E7%AD%86%E8%A8%98">筆記</a>
|
|
|
</li>
|
|
|
|
|
|
<li class="menu-item menu-item-type-taxonomy menu-item-object-category">
|
|
|
<a href="/categories/%E7%BE%A4%E6%9A%89">群暉</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
</ul>
|
|
|
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
<ul class="social-media-icons">
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20Social%20profile%20url%20in%20facebook" data-animate-hover="pulse" class="facebook" target="_blank">
|
|
|
<i class="fab fa-facebook-square" title="facebook"></i>
|
|
|
<span class="screen-reader-text">facebook</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20googleplus" data-animate-hover="pulse" class="gplus" target="_blank">
|
|
|
<i class="fab fa-google-plus-g" title="googleplus"></i>
|
|
|
<span class="screen-reader-text">googleplus</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="chang0206" data-animate-hover="pulse" class="twitter" target="_blank">
|
|
|
<i class="fab fa-twitter-square" title="twitter"></i>
|
|
|
<span class="screen-reader-text">twitter</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="chang0206" data-animate-hover="pulse" class="instagram" target="_blank">
|
|
|
<i class="fab fa-instagram" title="instagram"></i>
|
|
|
<span class="screen-reader-text">instagram</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="mailto:mc@hotshraingmy.info" data-animate-hover="pulse" class="email">
|
|
|
<i class="fas fa-envelope" title="email"></i>
|
|
|
<span class="screen-reader-text">email</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20linkedin" data-animate-hover="pulse" class="linkedin" target="_blank">
|
|
|
<i class="fab fa-linkedin-in" title="linkedin"></i>
|
|
|
<span class="screen-reader-text">linkedin</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20stackoverflow" data-animate-hover="pulse" class="stackoverflow" target="_blank">
|
|
|
<i class="fab fa-stack-overflow" title="stackoverflow"></i>
|
|
|
<span class="screen-reader-text">stackoverflow</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="changchichung" data-animate-hover="pulse" class="github" target="_blank">
|
|
|
<i class="fab fa-github" title="github"></i>
|
|
|
<span class="screen-reader-text">github</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20pinterest" data-animate-hover="pulse" class="pinterest" target="_blank">
|
|
|
<i class="fab fa-pinterest" title="pinterest"></i>
|
|
|
<span class="screen-reader-text">pinterest</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="https://h.cowbay.org/index.xml" data-animate-hover="pulse" class="rss" target="_blank">
|
|
|
<i class="fas fa-rss" title="rss"></i>
|
|
|
<span class="screen-reader-text">rss</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
</ul></div>
|
|
|
|
|
|
</div>
|
|
|
|
|
|
<div class="container">
|
|
|
<div id="title-info" class="title-info">
|
|
|
<div id='site-title' class='site-title'>
|
|
|
|
|
|
<a href="/"> MC部落 </a>
|
|
|
</div>
|
|
|
</div>
|
|
|
<button id="toggle-navigation" class="toggle-navigation">
|
|
|
<i class="fas fa-bars"></i>
|
|
|
</button>
|
|
|
|
|
|
<div id="menu-primary-tracks" class="menu-primary-tracks"></div>
|
|
|
<div id="menu-primary" class="menu-container menu-primary" role="navigation">
|
|
|
|
|
|
<p class="site-description">What’s the Worst That Could Happen?</p>
|
|
|
|
|
|
|
|
|
<div class="menu">
|
|
|
<ul id="menu-primary-items" class="menu-primary-items">
|
|
|
|
|
|
|
|
|
<li class='menu-item menu-item-type-custom menu-item-object-custom '>
|
|
|
<a href="https://h.cowbay.org/">Home</a>
|
|
|
|
|
|
</li>
|
|
|
|
|
|
<li class='menu-item menu-item-type-post_type menu-item-object-page '>
|
|
|
<a href="https://h.cowbay.org/about/">About</a>
|
|
|
|
|
|
</li>
|
|
|
|
|
|
<li class='menu-item menu-item-type-post_type menu-item-object-page '>
|
|
|
<a href="https://h.cowbay.org/contact/">Get in touch</a>
|
|
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
</header>
|
|
|
|
|
|
<div id="main" class="main" role="main">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div id="loop-container" class="loop-container">
|
|
|
|
|
|
<div class="post type-post status-publish format-standard has-post-thumbnail hentry category-design tag-design tag-standard-2 tag-tagalicious tag-travel entry full-without-featured odd excerpt-1">
|
|
|
|
|
|
<div class='featured-image lazy lazy-bg-image' data-background="https://h.cowbay.org/images/post-default-03.jpg">
|
|
|
</div>
|
|
|
|
|
|
<div class="entry-meta">
|
|
|
<span class="date">30 November</span> <span> / </span>
|
|
|
|
|
|
<span class="author">
|
|
|
<a href="https://github.com/changchichung" title="Posts by Eric Chang" rel="author">Eric Chang</a>
|
|
|
</span>
|
|
|
|
|
|
|
|
|
|
|
|
<span class="category">
|
|
|
<span> / </span>
|
|
|
|
|
|
<a href="/categories/%E7%AD%86%E8%A8%98">筆記</a>
|
|
|
</span>
|
|
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
<div class='entry-header'>
|
|
|
<h1 class='entry-title'> [筆記] 用 proxmox & Mellanox SFP 網卡土炮 10G LAB </h1>
|
|
|
</div>
|
|
|
<div class="entry-container">
|
|
|
<div class="entry-content">
|
|
|
<article>
|
|
|
<p>想做一個 10G 的 LAB 環境出來已經很久了。</p>
|
|
|
|
|
|
<p>只是礙於10G RJ45的卡太貴了，然後光纖的種類又太複雜</p>
|
|
|
|
|
|
<p>如果直接在淘寶購買，很怕會買錯(什麼LC/FC LC/LC 多模單模 單芯雙芯 SFP/SFP+ 又是什麼光模塊的一大堆規格)</p>
|
|
|
|
|
|
<p>所以一直沒有付諸行動。</p>
|
|
|
|
|
|
<p>硬體的工作很久沒碰了，剛好在蝦皮看到有個賣家在賣 mellanox 的X2網卡，以在台灣的價格來說，算很便宜的 (550)</p>
|
|
|
|
|
|
<p>聊了一下，跟他請教了關於線材、光纖模塊的問題，回答也都很快很到位</p>
|
|
|
|
|
|
<p>就直接下訂了兩張網卡、兩個光纖模塊、一條LC/LC 光纖線</p>
|
|
|
|
|
|
<p>就是到貨有點久，等了兩個禮拜左右，一直到昨天東西才寄到</p>
|
|
|
|
|
|
<p>今天就花了點時間測試一下</p>
|
|
|
|
|
|
<p>先上個圖！</p>
|
|
|
|
|
|
<p><img src="https://i.imgur.com/34MNFkW.png" alt="10G LAB" /></p>
|
|
|
|
|
|
<p>簡單說，就是有兩台機器，分別安裝 proxmox (一台是新裝的，另一台是本來就在線上的LAB用機器)以及光纖網卡</p>
|
|
|
|
|
|
<p>Mellanox 這張 X2 的卡， proxmox 5.1 / 5.2 可以直接抓到，所以不必另外安裝驅動程式</p>
|
|
|
|
|
|
<p>硬體安裝很順利，不過軟體的設定就碰到點麻煩了，所以才想說作個筆記..</p>
|
|
|
|
|
|
<h3 id="strong-font-color-red-必須作-vmbridge-才能指定這個網卡給vm用-font-strong"><strong><font color="red">必須作 vmbridge 才能指定這個網卡給VM用 </font></strong></h3>
|
|
|
|
|
|
<p>安裝好網卡，開機，透過proxmox的WEB界面設定好網卡的資料後，原本以為可以直接使用了</p>
|
|
|
|
|
|
<p>但是proxmox 會提示需要重新開機才能變更設定</p>
|
|
|
|
|
|
<p>可是重新開機後，我兩台怎麼都ping不到對方</p>
|
|
|
|
|
|
<p>在這之前，我已經用兩台 ubuntu 18.04 client 測試過了，只要設定好IP就可以直接通</p>
|
|
|
|
|
|
<p>所以在這邊碰到這個問題，我滿訝異的</p>
|
|
|
|
|
|
<p>可是看網卡的燈號，明明就有亮起來，應該是正常的呀</p>
|
|
|
|
|
|
<p>原來，在proxmox 中，新增了網卡，並不是直接就可以拿來用</p>
|
|
|
|
|
|
<p>要先設定好 bridge ，然後才能起新的VM、指定新設定的 vmbridge 給這個新起的機器使用</p>
|
|
|
|
|
|
<h3 id="strong-font-color-red-disk-cache-type-要改-font-strong"><strong><font color="red">Disk Cache type 要改</font></strong></h3>
|
|
|
|
|
|
<p>設定了新的 vmbridge 之後，就可以在新VM的設定畫面中，指定網卡走這個界面出去</p>
|
|
|
|
|
|
<p>可是這樣做出來的VM ，一直無法開機</p>
|
|
|
|
|
|
<p>錯誤訊息如下</p>
|
|
|
|
|
|
<pre><code>kvm: -drive file=/zp/images/100/vm-100-disk-1.qcow2,if=none,id=drive-virtio0,format=qcow2,cache=none,aio=native,detect-zeroes=on: file system may not support O_DIRECT
|
|
|
kvm: -drive file=/zp/images/100/vm-100-disk-1.qcow2,if=none,id=drive-virtio0,format=qcow2,cache=none,aio=native,detect-zeroes=on: Could not open '/zp/images/100/vm-100-disk-1.qcow2': Invalid argument
|
|
|
TASK ERROR: start failed: command '/usr/bin/kvm -id 100 -name 123123 -chardev 'socket,id=qmp,path=/var/run/qemu-server/100.qmp,server,nowait' -mon 'chardev=qmp,mode=control' -pidfile /var/run/qemu-server/100.pid -daemonize -smbios 'type=1,uuid=da27a9ea-fd55-4542-b2a7-8d5b09bf7611' -smp '2,sockets=1,cores=2,maxcpus=2' -nodefaults -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' -vga std -vnc unix:/var/run/qemu-server/100.vnc,x509,password -cpu kvm64,+lahf_lm,+sep,+kvm_pv_unhalt,+kvm_pv_eoi,enforce -m 2048 -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' -iscsi 'initiator-name=iqn.1993-08.org.debian:01:b972d1ad783' -drive 'file=/zp/template/iso/ubuntu-18.04.1-live-server-amd64.iso,if=none,id=drive-ide2,media=cdrom,aio=threads' -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' -drive 'file=/zp/images/100/vm-100-disk-1.qcow2,if=none,id=drive-virtio0,format=qcow2,cache=none,aio=native,detect-zeroes=on' -device 'virtio-blk-pci,drive=drive-virtio0,id=virtio0,bus=pci.0,addr=0xa,bootindex=100' -netdev 'type=tap,id=net0,ifname=tap100i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' -device 'virtio-net-pci,mac=A2:EA:45:EE:17:25,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300'' failed: exit code 1
|
|
|
</code></pre>
|
|
|
|
|
|
<p>當然先去拜google，果然就看到了提示，需要把 磁碟的 Cache 從預設的 Default(No Cache) 改成 write through</p>
|
|
|
|
|
|
<p><img src="https://i.imgur.com/cmClmGd.png" alt="proxmox dish cache mode" /></p>
|
|
|
|
|
|
<p>為什麼？我也不知道..不知道是不是因為我把磁碟種類選成用 Virtio Block 的關係</p>
|
|
|
|
|
|
<p>總之呢，改完之後就可以了 …</p>
|
|
|
|
|
|
<h3 id="strong-font-color-red-必須手動設定路由-font-strong"><strong><font color="red">必須手動設定路由</font></strong></h3>
|
|
|
|
|
|
<h4 id="update">Update</h4>
|
|
|
|
|
|
<pre><code>這邊可能是我有誤解，應該不需要先在pve 本機設定 10G網卡的 IP
|
|
|
直接進web console 去設定 vmbr1 就好了
|
|
|
</code></pre>
|
|
|
|
|
|
<p>設定好新的VM，開機、設定IP、重開機之後，會發現還是ping 不到另一台機器..(翻桌！)</p>
|
|
|
|
|
|
<p>只好又去拜google ，就看到了底下這篇</p>
|
|
|
|
|
|
<p><a href="https://forum.proxmox.com/threads/how-to-add-second-nic.40905/">https://forum.proxmox.com/threads/how-to-add-second-nic.40905/</a></p>
|
|
|
|
|
|
<p>大概點出了方向，必須要手動增加路由(感覺有點蠢)</p>
|
|
|
|
|
|
<p>像我的光纖網卡走的是 192.168.50.0/24 ，就要去把原有的192.168.50.0/24的路由給砍掉，然後再新增(是不是很蠢？)</p>
|
|
|
|
|
|
<pre><code>root@ssd:/etc/network# ip route del 192.168.50.0/24
|
|
|
root@ssd:/etc/network# ip route add 192.168.50.0/24 dev vmbr1
|
|
|
root@ssd:/etc/network# ip route
|
|
|
|
|
|
default via 192.168.11.253 dev vmbr0 onlink
|
|
|
192.168.11.0/24 dev vmbr0 proto kernel scope link src 192.168.11.215
|
|
|
192.168.50.0/24 dev vmbr1 scope link
|
|
|
root@ssd:/etc/network#
|
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
<p>OK ping 一下對面看能不能過</p>
|
|
|
|
|
|
<pre><code>root@ssd:/etc/network# ping 192.168.50.10
|
|
|
PING 192.168.50.10 (192.168.50.10) 56(84) bytes of data.
|
|
|
64 bytes from 192.168.50.10: icmp_seq=1 ttl=64 time=0.083 ms
|
|
|
64 bytes from 192.168.50.10: icmp_seq=2 ttl=64 time=0.067 ms
|
|
|
64 bytes from 192.168.50.10: icmp_seq=3 ttl=64 time=0.080 ms
|
|
|
64 bytes from 192.168.50.10: icmp_seq=4 ttl=64 time=0.126 ms
|
|
|
^C
|
|
|
--- 192.168.50.10 ping statistics ---
|
|
|
</code></pre>
|
|
|
|
|
|
<p>GOOD ！很好！通了一邊，另外一邊就照辦，兩邊都通了，就可以開始來測試速度了</p>
|
|
|
|
|
|
<p>p.s 這個路由不知道需不需要每次都手動增加，或者是有哪個config可以在開機時載入</p>
|
|
|
|
|
|
<p>沒記錯的話，應該是在 /etc/network/if-up.d/ 新增一個 route 檔案</p>
|
|
|
|
|
|
<p>不過這部份我不是很確定就是了</p>
|
|
|
|
|
|
<p>所以自己寫了一個 script 來用..</p>
|
|
|
|
|
|
<h3 id="strong-iperf-測試速度-strong"><strong>iperf 測試速度</strong></h3>
|
|
|
|
|
|
<p>在linux 上，我習慣用 iperf 來測試兩台主機的連接速度</p>
|
|
|
|
|
|
<p>兩邊都用 apt install iperf 裝好套件</p>
|
|
|
|
|
|
<p>然後找一台作為 server ，執行</p>
|
|
|
|
|
|
<pre><code>iperf -s
|
|
|
</code></pre>
|
|
|
|
|
|
<p>然後到另一台，去執行</p>
|
|
|
|
|
|
<pre><code>2018-11-30 15:36:58 [minion@ubuntu ~]$ iperf -d -t 600 -P 10 -c 192.168.50.200
|
|
|
WARNING: option -d is not valid for server mode
|
|
|
------------------------------------------------------------
|
|
|
Client connecting to 192.168.50.200, TCP port 5001
|
|
|
TCP window size: 85.0 KByte (default)
|
|
|
------------------------------------------------------------
|
|
|
[ 3] local 192.168.50.199 port 40980 connected with 192.168.50.200 port 5001
|
|
|
[ ID] Interval Transfer Bandwidth
|
|
|
[ 3] 0.0-600.0 sec 641 GBytes 9.18 Gbits/sec
|
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
<p>哈哈哈，有目有！測試速度來到了 9.18 Gbits 啊！ 就是一個爽啊！</p>
|
|
|
|
|
|
<p>記得那個 server IP 是你 VM 裡面設定的 IP，不是 proxmox 上面的</p>
|
|
|
|
|
|
<p>同場加映走 1Gb 網路的測試結果</p>
|
|
|
|
|
|
<pre><code>2018-11-30 16:39:37 [minion@ubuntu ~]$ iperf -d -t 600 -P 10 -c 192.168.11.171
|
|
|
WARNING: option -d is not valid for server mode
|
|
|
------------------------------------------------------------
|
|
|
Client connecting to 192.168.11.171, TCP port 5001
|
|
|
TCP window size: 85.0 KByte (default)
|
|
|
------------------------------------------------------------
|
|
|
[ 3] local 192.168.11.55 port 38582 connected with 192.168.11.171 port 5001
|
|
|
[ ID] Interval Transfer Bandwidth
|
|
|
[ 3] 0.0-600.0 sec 65.8 GBytes 941 Mbits/sec
|
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
<p>192.168.11.171 跟 192.168.50.200 是同一台機器，只是一個是10G網卡，一個是onboard的 1Gb 網卡</p>
|
|
|
|
|
|
<p>速度果然是提高了十倍呀，果然就是一個爽啊！！</p>
|
|
|
|
|
|
<h3 id="strong-實際開vm來測試看看-strong"><strong>實際開VM來測試看看</strong></h3>
|
|
|
|
|
|
<p><del>上面的測試，是兩台PVE HOST之間的連線測試</del></p>
|
|
|
|
|
|
<p>接下來，要實際測試在PVE中，建立新的VM，一台安裝FreeNAS 作為storage，另一台則是一般的client</p>
|
|
|
|
|
|
<p>步驟簡單來說，就是在ssd 這台PVE 建立一個新的VM，然後安裝FREENAS，並且提供NFS/iscsi 給另一台PVE Host作為storage來源</p>
|
|
|
|
|
|
<p>新增storage選NFS，填入必要資訊後，在這台主機上，建立一個新的VM，磁碟選擇剛剛連接的NFS</p>
|
|
|
|
|
|
<h5 id="strong-要特別注意-freenas的nfs-share的參數要改-strong"><strong>要特別注意，freenas的NFS Share的參數要改</strong></h5>
|
|
|
|
|
|
<p>在 mapuser/mapgroup這邊要改成 root/wheel 不然會有無法寫入的問題</p>
|
|
|
|
|
|
<p><img src="https://i.imgur.com/Yn7qYdK.png" alt="FREENAS NFS Sharing" /></p>
|
|
|
|
|
|
<p>安裝完之後，實際跑一下 dd 看看速度多少</p>
|
|
|
|
|
|
<pre><code>Last login: Mon Dec 3 03:10:54 2018
|
|
|
2018-12-03 03:15:03 [administrator@ubuntu ~]$ dd if=/dev/zero of=testfile bs=10240 count=1000000
|
|
|
1000000+0 records in
|
|
|
1000000+0 records out
|
|
|
10240000000 bytes (10 GB, 9.5 GiB) copied, 9.63458 s, 1.1 GB/s
|
|
|
</code></pre>
|
|
|
|
|
|
<pre><code>2018-12-03 03:17:28 [administrator@ubuntu ~]$ dd if=/dev/zero of=testfile bs=20480 count=1000000
|
|
|
1000000+0 records in
|
|
|
1000000+0 records out
|
|
|
20480000000 bytes (20 GB, 19 GiB) copied, 16.0786 s, 1.3 GB/s
|
|
|
</code></pre>
|
|
|
|
|
|
<pre><code>2018-12-03 03:17:50 [administrator@ubuntu ~]$ dd if=/dev/zero of=testfile bs=4096 count=1000000
|
|
|
1000000+0 records in
|
|
|
1000000+0 records out
|
|
|
4096000000 bytes (4.1 GB, 3.8 GiB) copied, 4.80629 s, 852 MB/s
|
|
|
2018-12-03 03:25:23 [administrator@ubuntu ~]$
|
|
|
</code></pre>
|
|
|
|
|
|
<p>可以看到不但大檔案速度都很快，就連小檔案(4096)居然也有852MB</p>
|
|
|
|
|
|
<p>我底層也不過就是四顆 SATA3 sandisk 240G SSD 而已啊</p>
|
|
|
|
|
|
<p>如果都換成PCI-E SSD ，嘿嘿…(流口水</p>
|
|
|
|
|
|
<hr />
|
|
|
|
|
|
<p>不過呢，這個也只是自己建的LAB玩玩看而已</p>
|
|
|
|
|
|
<p>真的要放到 production 環境去，我也還沒啥把握 (畢竟都是中古、二手、退役的產品拼湊起來的)</p>
|
|
|
|
|
|
<p>而且沒有10G Switch ，所以只能點對點連接</p>
|
|
|
|
|
|
<p>說不定等到對岸的 10G Switch 開始大降價 (我覺得 8 port SFP+ / NTD $2000 左右我應該就會出手了)</p>
|
|
|
|
|
|
<p>再來把10G 的環境弄完整一點！</p>
|
|
|
</article>
|
|
|
</div>
|
|
|
|
|
|
<div class='entry-meta-bottom'>
|
|
|
|
|
|
|
|
|
<div class="entry-categories"><p><span>Categories</span>
|
|
|
|
|
|
<a href="/categories/%E7%AD%86%E8%A8%98" title="View all posts in 筆記">筆記</a>
|
|
|
</p>
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
<div class="entry-tags"><p><span>Tags</span>
|
|
|
|
|
|
<a href="/tags/10g" title="View all posts tagged 10G">10G</a>
|
|
|
|
|
|
<a href="/tags/%E7%AD%86%E8%A8%98" title="View all posts tagged 筆記">筆記</a>
|
|
|
|
|
|
<a href="/tags/mellanox" title="View all posts tagged mellanox">mellanox</a>
|
|
|
|
|
|
|
|
|
</p></div> </div>
|
|
|
|
|
|
|
|
|
<div class="author-meta">
|
|
|
|
|
|
<div class="author">
|
|
|
|
|
|
<img alt='Eric Chang' src="https://www.gravatar.com/avatar/23f8ed94e007297499ac8df1641b3ff5?s=100&d=identicon" class='avatar avatar-72 photo' height='72' width='72'>
|
|
|
|
|
|
<span>
|
|
|
Written by:<a href="https://github.com/changchichung" title="Posts by Eric Chang" rel="author">Eric Chang</a> </span>
|
|
|
</div>
|
|
|
<div class="bio">
|
|
|
|
|
|
|
|
|
<p>塵世裡一個迷途小書僮</p>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<a class="facebook" target="_blank"
|
|
|
href="full%20Social%20profile%20url%20in%20facebook">
|
|
|
<i class="fab fa-facebook-f"
|
|
|
title="facebook icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
<a class="googleplus" target="_blank"
|
|
|
href="full%20profile%20url%20in%20googleplus">
|
|
|
<i class="fab fa-google-plus-g"
|
|
|
title="googleplus icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<a class="twitter" target="_blank"
|
|
|
href="chang0206">
|
|
|
<i class="fab fa-twitter-square"
|
|
|
title="twitter icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
<a class="linkedin" target="_blank"
|
|
|
href="full%20profile%20url%20in%20linkedin">
|
|
|
<i class="fab fa-linkedin"
|
|
|
title="linkedin icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
<a class="email" target="_blank"
|
|
|
href="mailto:mc@hotshraingmy.info">
|
|
|
<i class="fas fa-envelope"
|
|
|
title="email icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
<a class="instagram" target="_blank"
|
|
|
href="chang0206">
|
|
|
<i class="fab fa-instagram"
|
|
|
title="instagram icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
<a class="stackoverflow" target="_blank"
|
|
|
href="full%20profile%20url%20in%20stackoverflow">
|
|
|
<i class="fab fa-stack-overflow"
|
|
|
title="stackoverflow icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
<a class="github" target="_blank"
|
|
|
href="changchichung">
|
|
|
<i class="fab fa-github"
|
|
|
title="github icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<a class="pinterest" target="_blank"
|
|
|
href="full%20profile%20url%20in%20pinterest">
|
|
|
<i class="fab fa-pinterest"
|
|
|
title="pinterest icon"></i>
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
</div>
|
|
|
|
|
|
<section id="comments" class="comments">
|
|
|
<div id="disqus_thread"></div>
|
|
|
<script type="application/javascript">
|
|
|
var disqus_config = function () {
|
|
|
|
|
|
|
|
|
|
|
|
};
|
|
|
(function() {
|
|
|
if (["localhost", "127.0.0.1"].indexOf(window.location.hostname) != -1) {
|
|
|
document.getElementById('disqus_thread').innerHTML = 'Disqus comments not available by default when the website is previewed locally.';
|
|
|
return;
|
|
|
}
|
|
|
var d = document, s = d.createElement('script'); s.async = true;
|
|
|
s.src = '//' + "h-cowbay-org-1" + '.disqus.com/embed.js';
|
|
|
s.setAttribute('data-timestamp', +new Date());
|
|
|
(d.head || d.body).appendChild(s);
|
|
|
})();
|
|
|
</script>
|
|
|
<noscript>Please enable JavaScript to view the <a href="https://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
|
|
|
<a href="https://disqus.com" class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</section>
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
|
|
|
<footer id="site-footer" class="site-footer" role="contentinfo">
|
|
|
<h1>
|
|
|
|
|
|
<a href=""> MC部落 </a>
|
|
|
|
|
|
</h1>
|
|
|
|
|
|
|
|
|
<p class="site-description">What’s the Worst That Could Happen?</p>
|
|
|
|
|
|
|
|
|
<div id="menu-footer" class="menu-container menu-footer" role="navigation">
|
|
|
<div class="menu">
|
|
|
|
|
|
<ul id="menu-footer-items" class="menu-footer-items">
|
|
|
|
|
|
</ul>
|
|
|
|
|
|
</div> </div>
|
|
|
|
|
|
<ul class="social-media-icons">
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a class="facebook" target="_blank"
|
|
|
href="full%20Social%20profile%20url%20in%20facebook" >
|
|
|
<i class="fab fa-facebook-f" title="facebook"></i>
|
|
|
<span class="screen-reader-text">facebook</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a class="googleplus" target="_blank"
|
|
|
href="full%20profile%20url%20in%20googleplus" >
|
|
|
<i class="fab fa-google-plus-g" title="googleplus"></i>
|
|
|
<span class="screen-reader-text">googleplus</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="chang0206" class="twitter" target="_blank">
|
|
|
<i class="fab fa-twitter-square" title="twitter"></i>
|
|
|
<span class="screen-reader-text">twitter</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="chang0206" class="instagram" target="_blank">
|
|
|
<i class="fab fa-instagram" title="instagram"></i>
|
|
|
<span class="screen-reader-text">instagram</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="mailto:mc@hotshraingmy.info" class="email">
|
|
|
<i class="fas fa-envelope" title="email"></i>
|
|
|
<span class="screen-reader-text">email</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20linkedin" class="linkedin" target="_blank">
|
|
|
<i class="fab fa-linkedin-in" title="linkedin"></i>
|
|
|
<span class="screen-reader-text">linkedin</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20stackoverflow" class="stackoverflow" target="_blank">
|
|
|
<i class="fab fa-stack-overflow" title="stackoverflow"></i>
|
|
|
<span class="screen-reader-text">stackoverflow</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="changchichung" class="github" target="_blank">
|
|
|
<i class="fab fa-github" title="github"></i>
|
|
|
<span class="screen-reader-text">github</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="full%20profile%20url%20in%20pinterest" class="pinterest" target="_blank">
|
|
|
<i class="fab fa-pinterest" title="pinterest"></i>
|
|
|
<span class="screen-reader-text">pinterest</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
<li>
|
|
|
<a href="https://h.cowbay.org/index.xml" data-animate-hover="pulse" class="rss" target="_blank">
|
|
|
<i class="fas fa-rss" title="rss"></i>
|
|
|
<span class="screen-reader-text">rss</span>
|
|
|
</a>
|
|
|
</li>
|
|
|
|
|
|
|
|
|
</ul> <div class="design-credit">
|
|
|
|
|
|
<p>© 2018 Göran Svensson</p>
|
|
|
|
|
|
<p>Nederburg Hugo Theme by <a href="https://appernetic.io">Appernetic</a>.</p>
|
|
|
|
|
|
<p>A port of Tracks by Compete Themes.</p>
|
|
|
|
|
|
</div>
|
|
|
</footer>
|
|
|
|
|
|
</div>
|
|
|
<script src="https://h.cowbay.org/js/jquery.min.js"></script>
|
|
|
<script src="https://h.cowbay.org/js/jquerymigrate.js"></script>
|
|
|
<script src="https://h.cowbay.org/js/production.min.js?v=1577410044"></script>
|
|
|
|
|
|
</body>
|
|
|
</html>
|
