So far have not been able to get my pc direct connected to my server and I am hoping someone can see where I have gone wrong or what I have missed. I have searched through many posts and spent some time trying everything that I know. Windows is showing that the adapter is connected
ubuntu is showing the adapter as down until i run "ip link set enp4s0 up" on ubuntu the 10gbe adapter that i am have this issue with is enp4s0
I have 2 machines connected with 2 nics each.
One Gbe adapter connects to the wider lan 192.168.0.0, and a 10gb spf+ Mellanox connectx-3 connecting directly between the machines on the 192.168.2.0 network. Both machines have matching Mellanox cards and transceivers
I am using these transceivers FS for Mellanox MFM1T02A-SR Compatible 10GBASE-SR SFP+
Win 10 network settings
static ip
ip addr: 192.168.2.2/24
gateway:
dns:
Ubuntu 20.04 network settings
root@budnas:/home/bud# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:25:90:a6:ba:00 brd ff:ff:ff:ff:ff:ff
altname enp9s0
inet 192.168.0.200/24 brd 192.168.0.255 scope global dynamic eno1
valid_lft 85824sec preferred_lft 85824sec
inet6 2600:8801:b000:3200:225:90ff:fea6:ba00/64 scope global dynamic mngtmpaddr noprefixroute
valid_lft 86399sec preferred_lft 86399sec
inet6 fe80::225:90ff:fea6:ba00/64 scope link
valid_lft forever preferred_lft forever
3: eno2: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc fq_codel state DOWN group default qlen 1000
link/ether 00:25:90:a6:ba:01 brd ff:ff:ff:ff:ff:ff
altname enp10s0
4: enp4s0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether e4:1d:2d:4f:df:c0 brd ff:ff:ff:ff:ff:ff
root@budnas:/home/bud# ip link set enp4s0 up
root@budnas:/home/bud# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:25:90:a6:ba:00 brd ff:ff:ff:ff:ff:ff
altname enp9s0
inet 192.168.0.200/24 brd 192.168.0.255 scope global dynamic eno1
valid_lft 85646sec preferred_lft 85646sec
inet6 2600:8801:b000:3200:225:90ff:fea6:ba00/64 scope global dynamic mngtmpaddr noprefixroute
valid_lft 86393sec preferred_lft 86393sec
inet6 fe80::225:90ff:fea6:ba00/64 scope link
valid_lft forever preferred_lft forever
3: eno2: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc fq_codel state DOWN group default qlen 1000
link/ether 00:25:90:a6:ba:01 brd ff:ff:ff:ff:ff:ff
altname enp10s0
4: enp4s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether e4:1d:2d:4f:df:c0 brd ff:ff:ff:ff:ff:ff
inet6 fe80::e61d:2dff:fe4f:dfc0/64 scope link
valid_lft forever preferred_lft forever
root@budnas:/home/bud# nano /etc/netplan/00-installer-config.yaml
version: 2
renderer: networkd
ethernets:
eno1:
dhcp4: true
dhcp6: true
eno2:
optional: true
enp9s0:
optional: true
dhcp4: true
dhcp6: true
enp10s0:
optional: true
dhcp4: true
dhcp6: true
enps4s0:
optional: true
addresses: ['192.168.2.3/24']
root@budnas:/home/bud# netplan generate
root@budnas:/home/bud# netplan apply
root@budnas:/home/bud# ip route
default via 192.168.0.1 dev eno1 proto dhcp src 192.168.0.200 metric 100
192.168.0.0/24 dev eno1 proto kernel scope link src 192.168.0.200
192.168.0.1 dev eno1 proto dhcp scope link src 192.168.0.200 metric 100
root@budnas:/home/bud# flint -d /dev/mst/mt4099_pci_cr0 query
Image type: FS2
FW Version: 2.42.5000
FW Release Date: 5.9.2017
Product Version: 02.42.50.00
Rom Info: type=PXE version=3.4.752
Device ID: 4099
Description: Node Port1 Port2 Sys image
GUIDs: ffffffffffffffff ffffffffffffffff ffffffffffffffff ffffffffffffffff
MACs: e41d2d4fdfc0 e41d2d4fdfc1
VSD:
PSID: MT_1170110023
root@budnas:/home/bud# lshw -c network
-network
description: Ethernet interface
product: MT27500 Family [ConnectX-3]
vendor: Mellanox Technologies
physical id: 0
bus info: pci@0000:04:00.0
logical name: enp4s0
version: 00
serial: e4:1d:2d:4f:df:c0
size: 10Gbit/s
width: 64 bits
clock: 33MHz
capabilities: pm vpd msix pciexpress bus_master cap_list rom ethernet physical fibre
configuration: autonegotiation=off broadcast=yes driver=mlx4_en driverversion=4.0-0 duplex=full firmware=2.42.5000 latency=0 link=yes multicast=yes port=fibre speed=10Gbit/s
resources: irq:35 memory:fb300000-fb3fffff memory:f4000000-f47fffff memory:fb200000-fb2fffff
-network
description: Ethernet interface
product: 82574L Gigabit Network Connection
vendor: Intel Corporation
physical id: 0
bus info: pci@0000:09:00.0
logical name: eno1
version: 00
serial: 00:25:90:a6:ba:00
size: 1Gbit/s
capacity: 1Gbit/s
width: 32 bits
clock: 33MHz
capabilities: pm msi pciexpress msix bus_master cap_list ethernet physical tp 10bt 10bt-fd 100bt 100bt-fd 1000bt-fd autonegotiation
configuration: autonegotiation=on broadcast=yes driver=e1000e driverversion=5.13.0-39-generic duplex=full firmware=2.1-2 ip=192.168.0.200 latency=0 link=yes multicast=yes port=twisted pair speed=1Gbit/s
resources: irq:18 memory:fb500000-fb51ffff ioport:c000(size=32) memory:fb520000-fb523fff
*-network
description: Ethernet interface
product: 82574L Gigabit Network Connection
vendor: Intel Corporation
physical id: 0
bus info: pci@0000:0a:00.0
logical name: eno2
version: 00
serial: 00:25:90:a6:ba:01
capacity: 1Gbit/s
width: 32 bits
clock: 33MHz
capabilities: pm msi pciexpress msix bus_master cap_list ethernet physical tp 10bt 10bt-fd 100bt 100bt-fd 1000bt-fd autonegotiation
configuration: autonegotiation=on broadcast=yes driver=e1000e driverversion=5.13.0-39-generic firmware=2.1-2 latency=0 link=no multicast=yes port=twisted pair
resources: irq:19 memory:fb400000-fb41ffff ioport:b000(size=32) memory:fb420000-fb423fff
@Preston Maness Good idea but I still can not successfully ping.
Windows routing table
C:\Windows\system32>route -p add 192.168.2.0 mask 255.255.255.0 192.168.2.2
OK!
C:\Windows\system32>route print
Interface List
6...24 8a 07 5d a1 20 ......Mellanox ConnectX-3 Ethernet Adapter
7...2c f0 5d 9f 0d df ......Realtek PCIe 2.5GbE Family Controller
17...0a 00 27 00 00 11 ......VirtualBox Host-Only Ethernet Adapter
15...00 50 56 c0 00 01 ......VMware Virtual Ethernet Adapter for VMnet1
4...c8 e2 65 f8 dc 38 ......Bluetooth Device (Personal Area Network)
1...........................Software Loopback Interface 1
IPv4 Route Table
Active Routes:
Network Destination Netmask Gateway Interface Metric
0.0.0.0 0.0.0.0 192.168.0.1 192.168.0.27 25
127.0.0.0 255.0.0.0 On-link 127.0.0.1 331
127.0.0.1 255.255.255.255 On-link 127.0.0.1 331
127.255.255.255 255.255.255.255 On-link 127.0.0.1 331
192.168.0.0 255.255.255.0 On-link 192.168.0.27 281
192.168.0.27 255.255.255.255 On-link 192.168.0.27 281
192.168.0.255 255.255.255.255 On-link 192.168.0.27 281
192.168.2.0 255.255.255.0 On-link 192.168.2.2 271
192.168.2.2 255.255.255.255 On-link 192.168.2.2 271
192.168.2.255 255.255.255.255 On-link 192.168.2.2 271
192.168.56.0 255.255.255.0 On-link 192.168.56.1 281
192.168.56.1 255.255.255.255 On-link 192.168.56.1 281
192.168.56.255 255.255.255.255 On-link 192.168.56.1 281
192.168.248.0 255.255.255.0 On-link 192.168.248.1 291
192.168.248.1 255.255.255.255 On-link 192.168.248.1 291
192.168.248.255 255.255.255.255 On-link 192.168.248.1 291
224.0.0.0 240.0.0.0 On-link 127.0.0.1 331
224.0.0.0 240.0.0.0 On-link 192.168.56.1 281
224.0.0.0 240.0.0.0 On-link 192.168.248.1 291
224.0.0.0 240.0.0.0 On-link 192.168.0.27 281
224.0.0.0 240.0.0.0 On-link 192.168.2.2 271
255.255.255.255 255.255.255.255 On-link 127.0.0.1 331
255.255.255.255 255.255.255.255 On-link 192.168.56.1 281
255.255.255.255 255.255.255.255 On-link 192.168.248.1 291
255.255.255.255 255.255.255.255 On-link 192.168.0.27 281
255.255.255.255 255.255.255.255 On-link 192.168.2.2 271
===========================================================================
Persistent Routes:
Network Address Netmask Gateway Address Metric
192.168.2.0 255.255.255.0 192.168.2.2 1
===========================================================================
IPv6 Route Table
Active Routes:
If Metric Network Destination Gateway
7 281 ::/0 fe80::96a6:7eff:fe1e:8976
1 331 ::1/128 On-link
7 281 2600:8801:b000:3200::/56 fe80::96a6:7eff:fe1e:8976
7 281 2600:8801:b000:3200::/64 On-link
7 281 2600:8801:b000:3200:796a:e24a:3c16:4e89/128
On-link
7 281 2600:8801:b000:3200:b18e:3579:377d:ce14/128
On-link
17 281 fe80::/64 On-link
15 291 fe80::/64 On-link
7 281 fe80::/64 On-link
6 271 fe80::/64 On-link
17 281 fe80::c1f:2e81:f19f:281d/128
On-link
15 291 fe80::4487:9c20:68a9:f2c6/128
On-link
7 281 fe80::b18e:3579:377d:ce14/128
On-link
6 271 fe80::e086:b8e9:d6a2:68ca/128
On-link
1 331 ff00::/8 On-link
17 281 ff00::/8 On-link
15 291 ff00::/8 On-link
7 281 ff00::/8 On-link
6 271 ff00::/8 On-link
===========================================================================
Persistent Routes:
None
new ubuntu routing table
bud@budnas:~$ ip route
default via 192.168.0.1 dev eno1 proto dhcp src 192.168.0.200 metric 100
192.168.0.0/24 dev eno1 proto kernel scope link src 192.168.0.200
192.168.0.1 dev eno1 proto dhcp scope link src 192.168.0.200 metric 100
192.168.2.0/24 dev enp4s0 scope link
192.168.2.2 dev enp4s0 scope link
