FC4 network is slow during file transfers

haxorwear · #1 27th October 2005, 08:17 AM

I'm having some major issues with a FC4 box I've recently built. The machine is fast -- dual 3.6ghz 2m cache xeons w/ 4gb memory, dual gig-e ports, etc... The network externally facing is 100mb half-duplex (auto-neg with the Cisco switch it is plugged into).

I try to use winscp to transfer a 10 meg file to the system averaging around 5 kbytes/sec, if it manages to finish at all before stalling out. Transferring the same file to a redhat system on the same switch is pushing 700+kbytes/sec before it finishes. Same file, same source system transferring the file, and same application (WinSCP) transferring the file over the same network. I have iptables completely unloaded from the system (modules not loaded, iptables services stopped, etc).

I've done the following from reading these forums (search2win) and searching on the net extensively...

added this to modprobe.conf:
alias net-pf-10 off
alias ipv6 off

added "NETWORKING_IPV6=no" to /etc/sysconfig/network

made sure the network config (gui tool) shows ipv6 off on all interfaces.

I know I've tweaked a few other things in attempting to fix this, but nothing has seemed to make a difference.

Ideas? Is FC4 just flawed somewhere in the network stack? I noted the same results using ftp as well.

-Tim

sej7278 · #2 27th October 2005, 09:59 AM

it's fedora punishing you for having such a nice pc

are the nics intel pro 100/1000 ct's by any chance?

i found mine would crash the machine if i put too much traffic over it, although i had an ok transfer speed on the 100baset switch. i replaced it with a 7usd Realtek RTL-8169 and it works fine now, guess the intel drivers were broken.

other than that, maybe you've got full duplex configured and the switch can't hack it, crap cables maybe?

haxorwear · #3 27th October 2005, 11:59 PM

Quote:

Originally Posted by sej7278

it's fedora punishing you for having such a nice pc

are the nics intel pro 100/1000 ct's by any chance?

i found mine would crash the machine if i put too much traffic over it, although i had an ok transfer speed on the 100baset switch. i replaced it with a 7usd Realtek RTL-8169 and it works fine now, guess the intel drivers were broken.

other than that, maybe you've got full duplex configured and the switch can't hack it, crap cables maybe?

The nics are Intel Pro 1000s, wired up via CAT6 (good cabling) to our gig-e 48 port HP switch.

e1000: eth0: e1000_probe: Intel(R) PRO/1000 Network Connection
e1000: eth1: e1000_probe: Intel(R) PRO/1000 Network Connection

haxorwear · #4 28th October 2005, 07:15 PM

Here's an strace from the rsh command, you can see the strace stop up for a second before continuing...

[11:10:49 - root@xxxx:~]strace rsh node2 uname -a
execve("/usr/bin/rsh", ["rsh", "node2", "uname", "-a"], [/* 19 vars */]) = 0
brk(0) = 0x80003000
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa7000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=60812, ...}) = 0
old_mmap(NULL, 60812, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7f98000
close(3) = 0
open("/lib/libcrypt.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\3 60\7H"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=27660, ...}) = 0
old_mmap(NULL, 184604, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7f6a000
old_mmap(0xb7f6f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0xb7f6f000
old_mmap(0xb7f71000, 155932, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7f71000
close(3) = 0
open("/lib/libutil.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\2 60|\310"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=15080, ...}) = 0
old_mmap(NULL, 12432, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7f66000
old_mmap(0xb7f68000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0xb7f68000
close(3) = 0
open("/lib/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\n _B\000"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1489572, ...}) = 0
old_mmap(NULL, 1219548, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e3c000
old_mmap(0xb7f60000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x124000) = 0xb7f60000
old_mmap(0xb7f64000, 7132, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7f64000
close(3) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7e3b000
set_thread_area({entry_number:-1 -> 6, base_addr:0xb7e3b9e0, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
mprotect(0xb7f60000, 8192, PROT_READ) = 0
mprotect(0xb7f68000, 4096, PROT_READ) = 0
mprotect(0xb7f6f000, 4096, PROT_READ) = 0
mprotect(0x409000, 4096, PROT_READ) = 0
munmap(0xb7f98000, 60812) = 0
getuid32() = 0
brk(0) = 0x80003000
brk(0x80024000) = 0x80024000
socket(PF_FILE, SOCK_STREAM, 0) = 3
fcntl64(3, F_GETFL) = 0x2 (flags O_RDWR)
fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(3, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
socket(PF_FILE, SOCK_STREAM, 0) = 3
fcntl64(3, F_GETFL) = 0x2 (flags O_RDWR)
fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(3, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
open("/etc/nsswitch.conf", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=1686, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa6000
read(3, "#\n# /etc/nsswitch.conf\n#\n# An ex"..., 4096) = 1686
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb7fa6000, 4096) = 0
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=60812, ...}) = 0
old_mmap(NULL, 60812, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7f98000
close(3) = 0
open("/lib/libnss_files.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0 \0 \33\0\000"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=46552, ...}) = 0
old_mmap(NULL, 41616, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e30000
old_mmap(0xb7e39000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0xb7e39000
close(3) = 0
mprotect(0xb7e39000, 4096, PROT_READ) = 0
munmap(0xb7f98000, 60812) = 0
open("/etc/passwd", O_RDONLY) = 3
fcntl64(3, F_GETFD) = 0
fcntl64(3, F_SETFD, FD_CLOEXEC) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=2107, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa6000
read(3, "root:x:0:0:root:/root:/bin/bash\n"..., 4096) = 2107
close(3) = 0
munmap(0xb7fa6000, 4096) = 0
open("/etc/services", O_RDONLY) = 3
fcntl64(3, F_GETFD) = 0
fcntl64(3, F_SETFD, FD_CLOEXEC) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=20385, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa6000
read(3, "# /etc/services:\n# $Id: services"..., 4096) = 4096
read(3, "otocol\nntp\t\t123/tcp\nntp\t\t123/udp"..., 4096) = 4096
read(3, "67/udp\nrsync\t\t873/tcp\t\t\t\t# rsync"..., 4096) = 4096
close(3) = 0
munmap(0xb7fa6000, 4096) = 0
getpid() = 16366
gettimeofday({1130523068, 645225}, NULL) = 0
open("/etc/resolv.conf", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa6000
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb7fa6000, 4096) = 0
uname({sys="Linux", node="pcm3m.accentopto.com", ...}) = 0
socket(PF_FILE, SOCK_STREAM, 0) = 3
fcntl64(3, F_GETFL) = 0x2 (flags O_RDWR)
fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(3, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
socket(PF_FILE, SOCK_STREAM, 0) = 3
fcntl64(3, F_GETFL) = 0x2 (flags O_RDWR)
fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(3, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
open("/etc/host.conf", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=17, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa6000
read(3, "order hosts,bind\n", 4096) = 17
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb7fa6000, 4096) = 0
open("/etc/hosts", O_RDONLY) = 3
fcntl64(3, F_GETFD) = 0
fcntl64(3, F_SETFD, FD_CLOEXEC) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=1826, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa6000
read(3, "# Do not remove the following li"..., 4096) = 1826
close(3) = 0
munmap(0xb7fa6000, 4096) = 0
open("/etc/hosts", O_RDONLY) = 3
fcntl64(3, F_GETFD) = 0
fcntl64(3, F_SETFD, FD_CLOEXEC) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=1826, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa6000
read(3, "# Do not remove the following li"..., 4096) = 1826
close(3) = 0
munmap(0xb7fa6000, 4096) = 0
rt_sigprocmask(SIG_BLOCK, [URG], [], 8) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 3
bind(3, {sa_family=AF_INET, sin_port=htons(1023), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
fcntl64(3, F_SETOWN, 16366) = 0
connect(3, {sa_family=AF_INET, sin_port=htons(514), sin_addr=inet_addr("192.168.101.2")}, 16) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 4
bind(4, {sa_family=AF_INET, sin_port=htons(1022), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
listen(4, 1) = 0
write(3, "1022\0", 5) = 5
poll([{fd=3, events=POLLIN}, {fd=4, events=POLLIN, revents=POLLIN}], 2, -1) = 1
accept(4, {sa_family=AF_INET, sin_port=htons(1023), sin_addr=inet_addr("192.168.101.2")}, [16]) = 5
close(4) = 0
writev(3, [{"root\0", 5}, {"root\0", 5}, {"uname -a\0", 9}], 3) = 19
read(3, "\0", 1) = 1
rt_sigprocmask(SIG_SETMASK, [], [URG], 8) = 0
setuid32(0) = 0
rt_sigprocmask(SIG_BLOCK, [INT QUIT TERM], [], 8) = 0
rt_sigaction(SIGINT, {SIG_IGN}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGINT, {0x80000cf4, [INT], SA_RESTART}, {SIG_IGN}, 8) = 0
rt_sigaction(SIGQUIT, {SIG_IGN}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGQUIT, {0x80000cf4, [QUIT], SA_RESTART}, {SIG_IGN}, 8) = 0
rt_sigaction(SIGTERM, {SIG_IGN}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGTERM, {0x80000cf4, [TERM], SA_RESTART}, {SIG_IGN}, 8) = 0
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGC HLD, child_tidptr=0xb7e3ba28) = 16367
ioctl(5, FIONBIO, [1]) = 0
ioctl(3, FIONBIO, [1]) = 0
rt_sigprocmask(SIG_SETMASK, [], [INT QUIT TERM], 8) = 0
select(6, [3 5], NULL, NULL, NULL) = 2 (in [3 5])
read(5, "", 8192) = 0
read(3, "Linux node2.pcm.cluster 2.6.13."..., 8192) = 95
write(1, "Linux node2.pcm.cluster 2.6.13."..., 95Linux node2.pcm.cluster 2.6.13.2 #1 SMP Mon Oct 10 11:59:57 PDT 2005 i686 i686 i386 GNU/Linux
) = 95
select(6, [3], NULL, NULL, NULL) = 1 (in [3])
read(3, "", 8192) = 0
kill(16367, SIGKILL) = 0
exit_group(0) = ?

n0xlf · #5 1st November 2005, 05:25 AM

Are you using jumbo frames? I had some strange problems with throughput until I tweaked my MTU to 8231 on the Fedora end of things...Certain Windows apps weren't happy with 9000 byte packets for some reason...

haxorwear · #6 1st November 2005, 06:49 PM

Nope, haven't gotten that far. I'm considering using jumbo frames, but right now mtus are 1500 still for std ethernet. The whole issue here is that the networking layer was slowing down RSH commands being issues to slaves on the private side of the network. This is part of a computing cluster. I thought in.rshd could be the issue, but it appears to be xinetd causing delays in everything it is running.

I ran strace -tt -p <pid of xinetd> and can see the delay very clearly...

[10:30:13 - root@xxxx:/etc]strace -tt -p 25273
Process 25273 attached - interrupt to quit
10:30:16.295617 select(11, [3 5 6 8 9 10], NULL, NULL, NULL) = 1 (in [9])
10:30:18.629949 accept(9, {sa_family=AF_INET, sin_port=htons(1023), sin_addr=inet_addr("192.168.101.2")}, [16]) = 11
10:30:18.630160 time(NULL) = 1130869818
10:30:18.630305 clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGC HLD, child_tidptr=0xb7d6d708) = 25394
10:30:18.630987 time([1130869818]) = 1130869818
10:30:18.631163 time([1130869818]) = 1130869818
10:30:18.631296 stat64("/etc/localtime", {st_mode=S_IFREG|0644, st_size=1017, ...}) = 0
10:30:18.631489 stat64("/etc/localtime", {st_mode=S_IFREG|0644, st_size=1017, ...}) = 0
10:30:18.631665 stat64("/etc/localtime", {st_mode=S_IFREG|0644, st_size=1017, ...}) = 0
10:30:18.631857 send(7, "<86>Nov 1 10:30:18 xinetd[25273"..., 76, MSG_NOSIGNAL) = 76
10:30:18.632090 close(11) = 0
10:30:18.632208 select(11, [3 5 6 8 9 10], NULL, NULL, NULL) = ? ERESTARTNOHAND (To be restarted)
10:30:20.389131 --- SIGCHLD (Child exited) @ 0 (0) ---
10:30:20.389230 write(4, "\21", 1) = 1
10:30:20.389359 sigreturn() = ? (mask now [])
10:30:20.389530 select(11, [3 5 6 8 9 10], NULL, NULL, NULL) = 1 (in [3])
10:30:20.389678 ioctl(3, FIONREAD, [1]) = 0
10:30:20.389804 read(3, "\21", 1) = 1
10:30:20.389931 waitpid(-1, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], WNOHANG) = 25394
10:30:20.390069 close(-1) = -1 EBADF (Bad file descriptor)
10:30:20.390192 waitpid(-1, 0xbfc16bc8, WNOHANG) = -1 ECHILD (No child processes)
10:30:20.390315 select(11, [3 5 6 8 9 10], NULL, NULL, NULL <unfinished ...>
Process 25273 detached

As you can see, the delay is between these two lines:
10:30:18.632208 select(11, [3 5 6 8 9 10], NULL, NULL, NULL) = ? ERESTARTNOHAND (To be restarted)
10:30:20.389131 --- SIGCHLD (Child exited) @ 0 (0) ---

That's the .5-1.5 second delay I keep seeing in my rsh commands. It appears to be something with xinetd, but I can't tell what at the moment.

-Tim

themolecule · #7 21st May 2008, 06:26 PM

Did you ever find a resolution for this? I have the same problem.