PDA

View Full Version : Optimization request


rixey
Tue 3rd Jul '07, 5:18am
1. 2 Dedicated server consist of Apache and mySQL server.

2. Server Specs

Webserver:
2x AMD Opteron(tm) Processor 254
4GB DDR RAM
36.7GB SCSI HDD
CentOS 4.5
Apache 2.0.52
PHP 4.4.7

2x Dual-Core AMD Opteron(tm) Processor 2220
6GB DDR RAM
72.9GB SCSI HDD
CentOS 4.5
mySQL 4.1.22-standard

3. vBulletin 3.6.0

4. No innodb database in server (skip-innodb enabled)

5. RPM installed

6.

Web server

top - 03:05:04 up 4 days, 10:07, 1 user, load average: 0.71, 1.08, 1.46
Tasks: 144 total, 1 running, 143 sleeping, 0 stopped, 0 zombie
Cpu0 : 49.3% us, 3.7% sy, 0.0% ni, 43.3% id, 0.1% wa, 0.3% hi, 3.3% si
Cpu1 : 34.3% us, 2.0% sy, 0.0% ni, 61.9% id, 1.3% wa, 0.0% hi, 0.5% si
Mem: 4037668k total, 2445072k used, 1592596k free, 35408k buffers
Swap: 2040244k total, 192k used, 2040052k free, 408836k cached

mySQL server

top - 02:48:27 up 21 days, 35 min, 1 user, load average: 0.06, 0.14, 0.16
Tasks: 275 total, 1 running, 274 sleeping, 0 stopped, 0 zombie
Cpu0 : 9.2% us, 5.6% sy, 0.0% ni, 84.1% id, 1.1% wa, 0.0% hi, 0.0% si
Cpu1 : 1.0% us, 0.7% sy, 0.0% ni, 97.8% id, 0.5% wa, 0.0% hi, 0.0% si
Cpu2 : 0.8% us, 0.6% sy, 0.0% ni, 97.7% id, 0.9% wa, 0.0% hi, 0.0% si
Cpu3 : 0.4% us, 0.3% sy, 0.0% ni, 98.5% id, 0.9% wa, 0.0% hi, 0.0% si
Mem: 6231332k total, 5960732k used, 270600k free, 47276k buffers
Swap: 2040244k total, 192k used, 2040052k free, 5344604k cached
7. my.cnf

[mysqld]
ft_min_word_len = 3
datadir = /var/lib/mysql
safe-show-database
old_passwords
back_log = 75
skip-innodb
max_connections = 3000
key_buffer = 192M
myisam_sort_buffer_size = 64M
join_buffer_size = 1M
read_buffer_size = 1M
sort_buffer_size = 3M
table_cache = 2500
thread_cache_size = 384
wait_timeout = 45
connect_timeout = 10
tmp_table_size = 128M
max_heap_table_size = 128M
max_allowed_packet = 64M
max_connect_errors = 10
thread_concurrency = 4
read_rnd_buffer_size = 524288
bulk_insert_buffer_size = 8M
query_cache_limit = 3M
query_cache_size = 48M
query_cache_type = 1
query_prealloc_size = 131072
query_alloc_block_size = 65536
default-storage-engine = MyISAM

[mysql.server]
user = mysql
basedir = /var/lib

[mysqld_safe]
log-error = /var/log/mysqld-err.log
pid-file = /var/lib/mysql/mysql.pid
nice = -5
open_files_limit = 8192
socket = /var/lib/mysql/mysql.sock

[mysqldump]
quick
max_allowed_packet = 16M

[myisamchk]
ft_min_word_len = 3
key_buffer = 64M
sort_buffer = 64M
read_buffer = 16M
write_buffer = 16M

[mysqlhotcopy]
interactive-timeout
8. mysql stats

+---------------------------------+------------------------------------------+
| Variable_name | Value |
+---------------------------------+------------------------------------------+
| back_log | 75 |
| basedir | /usr/local/mysql4/ |
| binlog_cache_size | 32768 |
| bulk_insert_buffer_size | 8388608 |
| character_set_client | latin1 |
| character_set_connection | latin1 |
| character_set_database | latin1 |
| character_set_results | latin1 |
| character_set_server | latin1 |
| character_set_system | utf8 |
| character_sets_dir | /usr/local/mysql4/share/mysql/charsets/ |
| collation_connection | latin1_swedish_ci |
| collation_database | latin1_swedish_ci |
| collation_server | latin1_swedish_ci |
| concurrent_insert | ON |
| connect_timeout | 10 |
| datadir | /var/lib/mysql/ |
| date_format | %Y-%m-%d |
| datetime_format | %Y-%m-%d %H:%i:%s |
| default_week_format | 0 |
| delay_key_write | ON |
| delayed_insert_limit | 100 |
| delayed_insert_timeout | 300 |
| delayed_queue_size | 1000 |
| expire_logs_days | 0 |
| flush | OFF |
| flush_time | 0 |
| ft_boolean_syntax | + -><()~*:""&| |
| ft_max_word_len | 84 |
| ft_min_word_len | 3 |
| ft_query_expansion_limit | 20 |
| ft_stopword_file | (built-in) |
| group_concat_max_len | 1024 |
| have_archive | YES |
| have_bdb | NO |
| have_blackhole_engine | NO |
| have_compress | YES |
| have_crypt | YES |
| have_csv | NO |
| have_example_engine | NO |
| have_geometry | YES |
| have_innodb | DISABLED |
| have_isam | NO |
| have_merge_engine | YES |
| have_ndbcluster | NO |
| have_openssl | NO |
| have_query_cache | YES |
| have_raid | YES |
| have_rtree_keys | YES |
| have_symlink | YES |
| init_connect | |
| init_file | |
| init_slave | |
| innodb_additional_mem_pool_size | 1048576 |
| innodb_autoextend_increment | 8 |
| innodb_buffer_pool_awe_mem_mb | 0 |
| innodb_buffer_pool_size | 8388608 |
| innodb_data_file_path | |
| innodb_data_home_dir | |
| innodb_fast_shutdown | ON |
| innodb_file_io_threads | 4 |
| innodb_file_per_table | OFF |
| innodb_flush_log_at_trx_commit | 1 |
| innodb_flush_method | |
| innodb_force_recovery | 0 |
| innodb_lock_wait_timeout | 50 |
| innodb_locks_unsafe_for_binlog | OFF |
| innodb_log_arch_dir | |
| innodb_log_archive | OFF |
| innodb_log_buffer_size | 1048576 |
| innodb_log_file_size | 5242880 |
| innodb_log_files_in_group | 2 |
| innodb_log_group_home_dir | |
| innodb_max_dirty_pages_pct | 90 |
| innodb_max_purge_lag | 0 |
| innodb_mirrored_log_groups | 1 |
| innodb_open_files | 300 |
| innodb_table_locks | ON |
| innodb_thread_concurrency | 8 |
| interactive_timeout | 28800 |
| join_buffer_size | 1044480 |
| key_buffer_size | 201326592 |
| key_cache_age_threshold | 300 |
| key_cache_block_size | 1024 |
| key_cache_division_limit | 100 |
| language | /usr/local/mysql4/share/mysql/english/ |
| large_files_support | ON |
| lc_time_names | en_US |
| license | GPL |
| local_infile | ON |
| locked_in_memory | OFF |
| log | OFF |
| log_bin | OFF |
| log_error | |
| log_slave_updates | OFF |
| log_slow_queries | OFF |
| log_update | OFF |
| log_warnings | 1 |
| long_query_time | 10 |
| low_priority_updates | OFF |
| lower_case_file_system | OFF |
| lower_case_table_names | 0 |
| max_allowed_packet | 67107840 |
| max_binlog_cache_size | 4294967295 |
| max_binlog_size | 1073741824 |
| max_connect_errors | 10 |
| max_connections | 3000 |
| max_delayed_threads | 20 |
| max_error_count | 64 |
| max_heap_table_size | 134216704 |
| max_insert_delayed_threads | 20 |
| max_join_size | 4294967295 |
| max_length_for_sort_data | 1024 |
| max_prepared_stmt_count | 16382 |
| max_relay_log_size | 0 |
| max_seeks_for_key | 4294967295 |
| max_sort_length | 1024 |
| max_tmp_tables | 32 |
| max_user_connections | 0 |
| max_write_lock_count | 4294967295 |
| myisam_data_pointer_size | 4 |
| myisam_max_extra_sort_file_size | 2147483648 |
| myisam_max_sort_file_size | 2147483647 |
| myisam_recover_options | OFF |
| myisam_repair_threads | 1 |
| myisam_sort_buffer_size | 67108864 |
| myisam_stats_method | nulls_unequal |
| net_buffer_length | 16384 |
| net_read_timeout | 30 |
| net_retry_count | 10 |
| net_write_timeout | 60 |
| new | OFF |
| old_passwords | ON |
| open_files_limit | 15000 |
| pid_file | /var/lib/mysql/db.realestniggas.com.pid |
| port | 3306 |
| preload_buffer_size | 32768 |
| prepared_stmt_count | 0 |
| protocol_version | 10 |
| query_alloc_block_size | 65536 |
| query_cache_limit | 3145728 |
| query_cache_min_res_unit | 4096 |
| query_cache_size | 50331648 |
| query_cache_type | ON |
| query_cache_wlock_invalidate | OFF |
| query_prealloc_size | 131072 |
| range_alloc_block_size | 2048 |
| read_buffer_size | 1044480 |
| read_only | OFF |
| read_rnd_buffer_size | 520192 |
| relay_log_purge | ON |
| relay_log_space_limit | 0 |
| rpl_recovery_rank | 0 |
| secure_auth | OFF |
| server_id | 0 |
| skip_external_locking | ON |
| skip_networking | OFF |
| skip_show_database | OFF |
| slave_net_timeout | 3600 |
| slave_transaction_retries | 0 |
| slow_launch_time | 2 |
| socket | /var/lib/mysql/mysql.sock |
| sort_buffer_size | 3145720 |
| sql_mode | |
| sql_notes | OFF |
| sql_warnings | OFF |
| storage_engine | MyISAM |
| sync_binlog | 0 |
| sync_frm | ON |
| sync_replication | 0 |
| sync_replication_slave_id | 0 |
| sync_replication_timeout | 0 |
| system_time_zone | CDT |
| table_cache | 2500 |
| table_type | MyISAM |
| thread_cache_size | 384 |
| thread_stack | 126976 |
| time_format | %H:%i:%s |
| time_zone | SYSTEM |
| tmp_table_size | 134217728 |
| tmpdir | |
| transaction_alloc_block_size | 8192 |
| transaction_prealloc_size | 4096 |
| tx_isolation | REPEATABLE-READ |
| version | 4.1.22-standard |
| version_comment | MySQL Community Edition - Standard (GPL) |
| version_compile_machine | i686 |
| version_compile_os | pc-linux-gnu |
| wait_timeout | 45 |
+---------------------------------+------------------------------------------+

+----------------------------+------------+
| Variable_name | Value |
+----------------------------+------------+
| Aborted_clients | 185433 |
| Aborted_connects | 21 |
| Binlog_cache_disk_use | 0 |
| Binlog_cache_use | 0 |
| Bytes_received | 1047019626 |
| Bytes_sent | 2224167012 |
| Com_admin_commands | 941565 |
| Com_alter_db | 0 |
| Com_alter_table | 0 |
| Com_analyze | 0 |
| Com_backup_table | 0 |
| Com_begin | 0 |
| Com_change_db | 1034363 |
| Com_change_master | 0 |
| Com_check | 0 |
| Com_checksum | 0 |
| Com_commit | 0 |
| Com_create_db | 0 |
| Com_create_function | 0 |
| Com_create_index | 0 |
| Com_create_table | 0 |
| Com_dealloc_sql | 0 |
| Com_delete | 114497 |
| Com_delete_multi | 0 |
| Com_do | 0 |
| Com_drop_db | 0 |
| Com_drop_function | 0 |
| Com_drop_index | 0 |
| Com_drop_table | 0 |
| Com_drop_user | 0 |
| Com_execute_sql | 0 |
| Com_flush | 0 |
| Com_grant | 0 |
| Com_ha_close | 0 |
| Com_ha_open | 0 |
| Com_ha_read | 0 |
| Com_help | 0 |
| Com_insert | 508947 |
| Com_insert_select | 1660 |
| Com_kill | 0 |
| Com_load | 0 |
| Com_load_master_data | 0 |
| Com_load_master_table | 0 |
| Com_lock_tables | 0 |
| Com_optimize | 0 |
| Com_preload_keys | 0 |
| Com_prepare_sql | 0 |
| Com_purge | 0 |
| Com_purge_before_date | 0 |
| Com_rename_table | 0 |
| Com_repair | 0 |
| Com_replace | 77481 |
| Com_replace_select | 0 |
| Com_reset | 0 |
| Com_restore_table | 0 |
| Com_revoke | 0 |
| Com_revoke_all | 0 |
| Com_rollback | 0 |
| Com_savepoint | 0 |
| Com_select | 5320430 |
| Com_set_option | 1 |
| Com_show_binlog_events | 0 |
| Com_show_binlogs | 0 |
| Com_show_charsets | 0 |
| Com_show_collations | 0 |
| Com_show_column_types | 0 |
| Com_show_create_db | 0 |
| Com_show_create_table | 0 |
| Com_show_databases | 2 |
| Com_show_errors | 0 |
| Com_show_fields | 121 |
| Com_show_grants | 0 |
| Com_show_innodb_status | 0 |
| Com_show_keys | 0 |
| Com_show_logs | 0 |
| Com_show_master_status | 0 |
| Com_show_ndb_status | 0 |
| Com_show_new_master | 0 |
| Com_show_open_tables | 0 |
| Com_show_privileges | 0 |
| Com_show_processlist | 2275 |
| Com_show_slave_hosts | 0 |
| Com_show_slave_status | 0 |
| Com_show_status | 7 |
| Com_show_storage_engines | 0 |
| Com_show_tables | 3 |
| Com_show_variables | 5 |
| Com_show_warnings | 0 |
| Com_slave_start | 0 |
| Com_slave_stop | 0 |
| Com_stmt_close | 0 |
| Com_stmt_execute | 0 |
| Com_stmt_prepare | 0 |
| Com_stmt_reset | 0 |
| Com_stmt_send_long_data | 0 |
| Com_truncate | 0 |
| Com_unlock_tables | 0 |
| Com_update | 1477189 |
| Com_update_multi | 0 |
| Connections | 95102 |
| Created_tmp_disk_tables | 1733 |
| Created_tmp_files | 8557 |
| Created_tmp_tables | 424989 |
| Delayed_errors | 0 |
| Delayed_insert_threads | 0 |
| Delayed_writes | 0 |
| Flush_commands | 1 |
| Handler_commit | 0 |
| Handler_delete | 326471 |
| Handler_discover | 0 |
| Handler_read_first | 2131695 |
| Handler_read_key | 301521196 |
| Handler_read_next | 1829483722 |
| Handler_read_prev | 15136566 |
| Handler_read_rnd | 13077405 |
| Handler_read_rnd_next | 2008671743 |
| Handler_rollback | 0 |
| Handler_update | 3311490 |
| Handler_write | 1163891 |
| Key_blocks_not_flushed | 0 |
| Key_blocks_unused | 0 |
| Key_blocks_used | 174450 |
| Key_read_requests | 1230224853 |
| Key_reads | 4336389 |
| Key_write_requests | 1767298 |
| Key_writes | 1231073 |
| Max_used_connections | 207 |
| Not_flushed_delayed_rows | 0 |
| Open_files | 589 |
| Open_streams | 0 |
| Open_tables | 472 |
| Opened_tables | 478 |
| Qcache_free_blocks | 1123 |
| Qcache_free_memory | 45086776 |
| Qcache_hits | 3630820 |
| Qcache_inserts | 5303040 |
| Qcache_lowmem_prunes | 412103 |
| Qcache_not_cached | 17389 |
| Qcache_queries_in_cache | 1272 |
| Qcache_total_blocks | 3807 |
| Questions | 12170081 |
| Rpl_status | NULL |
| Select_full_join | 25 |
| Select_full_range_join | 1 |
| Select_range | 1509586 |
| Select_range_check | 0 |
| Select_scan | 452305 |
| Slave_open_temp_tables | 0 |
| Slave_retried_transactions | 0 |
| Slave_running | OFF |
| Slow_launch_threads | 0 |
| Slow_queries | 11 |
| Sort_merge_passes | 4360 |
| Sort_range | 1255224 |
| Sort_rows | 447832194 |
| Sort_scan | 110045 |
| Table_locks_immediate | 15165968 |
| Table_locks_waited | 57563 |
| Threads_cached | 130 |
| Threads_connected | 77 |
| Threads_created | 207 |
| Threads_running | 1 |
| Uptime | 127340 |
+----------------------------+------------+

Uptime: 127355 Threads: 83 Questions: 12170934 Slow queries: 11 Opens: 478 Flush tables: 1 Open tables: 472 Queries per second avg: 95.567
mysqladmin Ver 8.41 Distrib 4.1.20, for redhat-linux-gnu on x86_64
Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
This software comes with ABSOLUTELY NO WARRANTY. This is free software,
and you are welcome to modify and redistribute it under the GPL license

Server version 4.1.22-standard
Protocol version 10
Connection 192.168.1.3 via TCP/IP
TCP port 3306
Uptime: 1 day 11 hours 22 min 35 sec

Threads: 83 Questions: 12170935 Slow queries: 11 Opens: 478 Flush tables: 1 Open tables: 472 Queries per second avg: 95.567

mysqlreport
MySQL 4.1.22-standard uptime 1 11:25:8 Tue Jul 3 02:55:53 2007

__ Key __________________________________________________ _______________
Buffer used 170.36M of 192.00M %Used: 88.73
Current 192.00M %Usage: 100.00
Write hit 30.33%
Read hit 99.65%

__ Questions __________________________________________________ _________
Total 12.18M 95.5/s
DMS 7.51M 58.9/s %Total: 61.63
QC Hits 3.63M 28.5/s 29.83
Com_ 1.98M 15.5/s 16.26
-Unknown 942.33k 7.4/s 7.74
COM_QUIT 2.39k 0.0/s 0.02
Slow 11 0.0/s 0.00 %DMS: 0.00
DMS 7.51M 58.9/s 61.63
SELECT 5.32M 41.8/s 43.72 70.94
UPDATE 1.48M 11.6/s 12.14 19.69
INSERT 511.01k 4.0/s 4.20 6.81
DELETE 114.58k 0.9/s 0.94 1.53
REPLACE 77.54k 0.6/s 0.64 1.03
Com_ 1.98M 15.5/s 16.26
change_db 1.04M 8.1/s 8.50
admin_comma 942.22k 7.4/s 7.74
show_proces 2.27k 0.0/s 0.02

__ SELECT and Sort __________________________________________________ ___
Scan 452.66k 3.6/s %SELECT: 8.50
Range 1.51M 11.8/s 28.37
Full join 25 0.0/s 0.00
Range check 0 0/s 0.00
Full rng join 1 0.0/s 0.00
Sort scan 110.11k 0.9/s
Sort range 1.26M 9.9/s
Sort mrg pass 4.36k 0.0/s

__ Query Cache __________________________________________________ _______
Memory usage 5.74M of 48.00M %Used: 11.95
Block Fragmnt 24.59%
Hits 3.63M 28.5/s
Inserts 5.31M 41.6/s
Insrt:Prune 12.88:1 38.4/s
Hit:Insert 0.68:1

__ Table Locks __________________________________________________ _______
Waited 57.58k 0.5/s %Total: 0.38
Immediate 15.18M 119.0/s

__ Tables __________________________________________________ ____________
Open 472 of 2500 %Cache: 18.88
Opened 478 0.0/s

__ Connections __________________________________________________ _______
Max used 207 of 3000 %Max: 6.90
Total 95.17k 0.7/s

__ Created Temp __________________________________________________ ______
Disk table 1.73k 0.0/s
Table 425.25k 3.3/s
File 8.56k 0.1/s

__ Threads __________________________________________________ ___________
Running 1 of 78
Cached 129 of 384 %Hit: 99.78
Created 207 0.0/s
Slow 0 0/s

__ Aborted __________________________________________________ ___________
Clients 185.56k 1.5/s
Connects 22 0.0/s

__ Bytes __________________________________________________ _____________
Sent 2.37G 18.6k/s
Received 1.05G 8.2k/s
9. yes the only thing in server

10. not sure, user online counter been disabled but pretty sure its alot. cookie timeout is 900sec

11. http://72.232.84.130/info.php

12. httpd.conf
we tried 256 and the server reached its capacity then increased to 700++ also the same, now we're setting it to 1024.

<IfModule prefork.c>
StartServers 15
MinSpareServers 15
MaxSpareServers 60
ServerLimit 2048
MaxClients 1024
MaxRequestsPerChild 1000
</IfModule>

<IfModule worker.c>
StartServers 15
MaxClients 768
MinSpareThreads 15
MaxSpareThreads 60
ThreadsPerChild 35
MaxRequestsPerChild 1000
</IfModule>

<IfModule perchild.c>
NumServers 15
StartThreads 15
MinSpareThreads 15
MaxSpareThreads 20
MaxThreadsPerChild 20
MaxRequestsPerChild 1000
</IfModule>
13. no issue with big files

14.

Web server
Linux rn.adminz.net 2.6.9-55.ELsmp #1 SMP Wed May 2 14:04:42 EDT 2007 x86_64 x86_64 x86_64 GNU/Linux

core file size (blocks, -c) unlimited
data seg size (kbytes, -d) unlimited
file size (blocks, -f) unlimited
pending signals (-i) 1024
max locked memory (kbytes, -l) 32
max memory size (kbytes, -m) unlimited
open files (-n) 1024
pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200
stack size (kbytes, -s) unlimited
cpu time (seconds, -t) unlimited
max user processes (-u) 40960
virtual memory (kbytes, -v) unlimited
file locks (-x) unlimited

processor : 0
vendor_id : AuthenticAMD
cpu family : 15
model : 37
model name : AMD Opteron(tm) Processor 254
stepping : 1
cpu MHz : 2813.021
cache size : 1024 KB
fpu : yes
fpu_exception : yes
cpuid level : 1
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 syscall nx mmxext lm 3dnowext 3dnow pni
bogomips : 5631.64
TLB size : 1088 4K pages
clflush size : 64
cache_alignment : 64
address sizes : 40 bits physical, 48 bits virtual
power management: ts fid vid ttp

processor : 1
vendor_id : AuthenticAMD
cpu family : 15
model : 37
model name : AMD Opteron(tm) Processor 254
stepping : 1
cpu MHz : 2813.021
cache size : 1024 KB
fpu : yes
fpu_exception : yes
cpuid level : 1
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 syscall nx mmxext lm 3dnowext 3dnow pni
bogomips : 5625.11
TLB size : 1088 4K pages
clflush size : 64
cache_alignment : 64
address sizes : 40 bits physical, 48 bits virtual
power management: ts fid vid ttp

mySQL
Linux db.adminz.net 2.6.9-55.ELsmp #1 SMP Wed May 2 14:28:44 EDT 2007 i686 athlon i386 GNU/Linux

core file size (blocks, -c) unlimited
data seg size (kbytes, -d) unlimited
file size (blocks, -f) unlimited
pending signals (-i) 1024
max locked memory (kbytes, -l) 32
max memory size (kbytes, -m) unlimited
open files (-n) 1024
pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200
stack size (kbytes, -s) unlimited
cpu time (seconds, -t) unlimited
max user processes (-u) 107008
virtual memory (kbytes, -v) unlimited
file locks (-x) unlimited

processor : 0
vendor_id : AuthenticAMD
cpu family : 15
model : 65
model name : Dual-Core AMD Opteron(tm) Processor 2220
stepping : 3
cpu MHz : 2794.068
cache size : 1024 KB
physical id : 0
siblings : 2
core id : 0
cpu cores : 2
fdiv_bug : no
hlt_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 1
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht pni syscall nx mmxext fxsr_opt lm 3dnowext 3dnow pni
bogomips : 5589.44

processor : 1
vendor_id : AuthenticAMD
cpu family : 15
model : 65
model name : Dual-Core AMD Opteron(tm) Processor 2220
stepping : 3
cpu MHz : 2794.068
cache size : 1024 KB
physical id : 0
siblings : 2
core id : 1
cpu cores : 2
fdiv_bug : no
hlt_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 1
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht pni syscall nx mmxext fxsr_opt lm 3dnowext 3dnow pni
bogomips : 5585.85

processor : 2
vendor_id : AuthenticAMD
cpu family : 15
model : 65
model name : Dual-Core AMD Opteron(tm) Processor 2220
stepping : 3
cpu MHz : 2794.068
cache size : 1024 KB
physical id : 1
siblings : 2
core id : 0
cpu cores : 2
fdiv_bug : no
hlt_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 1
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht pni syscall nx mmxext fxsr_opt lm 3dnowext 3dnow pni
bogomips : 5585.97

processor : 3
vendor_id : AuthenticAMD
cpu family : 15
model : 65
model name : Dual-Core AMD Opteron(tm) Processor 2220
stepping : 3
cpu MHz : 2794.068
cache size : 1024 KB
physical id : 1
siblings : 2
core id : 1
cpu cores : 2
fdiv_bug : no
hlt_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 1
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht pni syscall nx mmxext fxsr_opt lm 3dnowext 3dnow pni
bogomips : 5586.16

rixey
Wed 4th Jul '07, 5:53am
Well, theres no problem with mySQL server it's just that I believe the problem is with apache. Maxclient keeps on maxing out and increasing it from 256 then 512 and now 1024. As it has the whole server for itself, setting it 1024 won't be a problem.

However, I'm not that sure on setting up the MPM to make use of CPU multi-threading. Also seems to be fine after increasing it to 1024 but time will tell if its sufficient or not.

Appreciate any feedback on this issue.

eva2000
Thu 5th Jul '07, 8:23am
maxclients keeps maxing out - how you determining this from apache error log or you getting 'too many connections' message from vB/mysql ? The higher you set maxclients the more memory apache server will use. Usually 150-256 value would suffice.

what about these 3 values outside of prefork tags in httpd.conf?

KeepAlive: On
MaxKeepAliveRequests: 100
KeepAliveTimeout: 4


You're using PHP 4.4.2 which is kind of out dated now so try the following in this exact order. You can ignore any of the suggestions that you have already done.

1. You need to install same mysql version on web server and then recompile / upgrade to PHP 4.4.7 after upgrading MySQL see further below for more details
2. Upgrade to Apache 2.0.59
3. Edit /etc/my.cnf and place the following mysql server settings in /etc/my.cnf and restart mysql server afterwards


[mysqld]
ft_min_word_len = 3
datadir = /var/lib/mysql
safe-show-database
old_passwords
back_log = 100
skip-innodb
max_connections = 650
key_buffer = 256M
myisam_sort_buffer_size = 64M
join_buffer_size = 2M
read_buffer_size = 2M
sort_buffer_size = 4M
table_cache = 2500
thread_cache_size = 384
wait_timeout = 45
connect_timeout = 10
tmp_table_size = 256M
max_heap_table_size = 256M
max_allowed_packet = 128M
max_connect_errors = 10
thread_concurrency = 8
read_rnd_buffer_size = 1M
bulk_insert_buffer_size = 8M
query_cache_limit = 8M
query_cache_size = 128M
query_cache_type = 1
query_prealloc_size = 131072
query_alloc_block_size = 65536
default-storage-engine = MyISAM

[mysql.server]
user = mysql
basedir = /var/lib

[mysqld_safe]
log-error = /var/log/mysqld-err.log
pid-file = /var/lib/mysql/mysql.pid
socket = /var/lib/mysql/mysql.sock
nice = -10
open_files_limit = 8192

[mysqldump]
quick
max_allowed_packet = 16M

[myisamchk]
ft_min_word_len = 3
key_buffer = 64M
sort_buffer = 64M
read_buffer = 16M
write_buffer = 16M

[mysqlhotcopy]
interactive-timeout


4. Install APC Cache lastest version from http://pecl.php.net/package/APC. Read install guide at http://www.vbulletin.com/forum/showthread.php?t=165367 - please remove Zend Optimizer from php.ini before installing APC Cache. OR try the alternative PHP caching software, Xcache which seems to be a bit faster than APC Cache - http://www.vbulletin.com/forum/showthread.php?t=213267

5. Upgrade to vB 3.6.7 http://www.vbulletin.com/forum/showthread.php?t=229950 if you're on vB 3.6.x

6. If you just upgraded to vB 3.5.x/3.6.x try to disable these 4 options:

Admin CP -> vBulletin Options -> Forums Home Page Options -> Display Logged in Users?

Admin CP -> vBulletin Options -> Forum Display Options (forumdisplay) -> Show Users Browsing Forums

Admin CP -> vBulletin Options -> Thread Display Options -> Show Users Browsing Thread

Admin CP -> vBulletin Options -> Message Searching Options -> Automatic Similar Thread search

7. Edit httpd.conf values to following and restart apache

what about these 3 values outside of prefork tags ?

KeepAlive: On
MaxKeepAliveRequests: 100
KeepAliveTimeout: 1

<IfModule prefork.c>
StartServers 15
MinSpareServers 15
MaxSpareServers 20
ServerLimit 2048
MaxClients 512
MaxRequestsPerChild 1000
</IfModule>

try maxclients 512 after the above suggestions are made

8. You may need to install same mysql version on web server, and recompile PHP with the --with-mysql=/path/to/mysql option to tell PHP to use the latest mysql 4.1.x client libraries i.e. if mysqldump/mysqlcheck/mysqladmin are located at /usr/bin/ or /usr/local/mysql/bin then you need to compile PHP with --with-mysql=/usr or --with-mysql=/usr/local/mysql - just had a few issues with one vB client have issues with PHP connecting to external mysql server until i did this.

9. If you have split web + db servers, ensure web server has dual network cards as outlined at http://www.vbulletin.org/forum/showthread.php?t=111191

rixey
Fri 14th Sep '07, 12:06am
1. These are in global setting, not under prefork tags.
KeepAlive On
MaxKeepAliveRequests 100
KeepAliveTimeout 1

2. MaxClient maxed out were identified by pstree where http total process equals the maxclient amount and of cause, the error_logs stated maxclient has reach consider raising it. We've reduced the MaxClient from 512 to 200 to avoid server lock out.

Perhaps I should post a new report again, I'll do that soon.

rixey
Tue 12th Feb '08, 3:32am
Here is the updated report and the server still maxing out it's maxclient. I think it has something to do with mysql due to slow query. When the web server hangs, I shutdown mysql and after a while its back again. I've appended slow query report at the end of this post.

I also considering to install apache2 with php 5 & mysql5. Will this make any significant change? or is it worth to try?

Let me know if u need more details.

1. Dedicate Server

2. Server Specs

CentOS 4.5
Apache 2.0.58
PHP 4.4.7
mysql 4.1.22-standard-log

WEB:
Dual AMD Opteron 254
4GB RAM
2x36GB SCSI RAID-1

SQL:
Dual-Core AMD Opteron 2220
6GB RAM
72.6GB SCSI RAID-1

3. vBulletin 3.6

4. Only vBulletin installed, no innodb

5. Precompiled sql

6. top stats

top - 01:06:05 up 3 days, 4:50, 2 users, load average: 2.19, 2.52, 2.76
Tasks: 106 total, 4 running, 101 sleeping, 0 stopped, 1 zombie
Cpu0 : 56.9% us, 3.2% sy, 0.0% ni, 34.5% id, 0.4% wa, 0.5% hi, 4.4% si
Cpu1 : 44.8% us, 2.0% sy, 0.0% ni, 52.2% id, 0.7% wa, 0.0% hi, 0.3% si
Mem: 4037668k total, 685672k used, 3351996k free, 28396k buffers
Swap: 2040244k total, 43448k used, 1996796k free, 39212k cached

SQL
top - 23:49:20 up 141 days, 9:36, 2 users, load average: 0.21, 0.21, 0.21
Tasks: 186 total, 1 running, 185 sleeping, 0 stopped, 0 zombie
Cpu0 : 6.7% us, 5.7% sy, 0.0% ni, 87.3% id, 0.3% wa, 0.0% hi, 0.0% si
Cpu1 : 0.0% us, 0.0% sy, 0.0% ni, 100.0% id, 0.0% wa, 0.0% hi, 0.0% si
Cpu2 : 0.0% us, 0.0% sy, 0.0% ni, 100.0% id, 0.0% wa, 0.0% hi, 0.0% si
Cpu3 : 0.7% us, 1.0% sy, 0.0% ni, 98.3% id, 0.0% wa, 0.0% hi, 0.0% si
Mem: 6231332k total, 6204508k used, 26824k free, 17792k buffers
Swap: 2040244k total, 192k used, 2040052k free, 5690508k cached
7. my.cnf

[mysqld]
ft_min_word_len = 3
datadir = /var/lib/mysql
safe-show-database
old_passwords
back_log = 100
skip-innodb
max_connections = 800
key_buffer = 256M
myisam_sort_buffer_size = 64M
join_buffer_size = 2M
read_buffer_size = 2M
sort_buffer_size = 4M
table_cache = 2500
thread_cache_size = 384
wait_timeout = 45
connect_timeout = 5
tmp_table_size = 256M
max_heap_table_size = 256M
max_allowed_packet = 128M
max_connect_errors = 10
thread_concurrency = 8
read_rnd_buffer_size = 1M
bulk_insert_buffer_size = 8M
query_cache_limit = 8M
query_cache_size = 128M
query_cache_type = 1
query_prealloc_size = 131072
query_alloc_block_size = 65536
default-storage-engine = MyISAM
long_query_time = 5

[mysql.server]
user = mysql
basedir = /var/lib

[mysqld_safe]
log-error = /var/log/mysqld-err.log
pid-file = /var/lib/mysql/mysql.pid
socket = /var/lib/mysql/mysql.sock
nice = -10
open_files_limit = 8192

[mysqldump]
quick
max_allowed_packet = 16M

[myisamchk]
ft_min_word_len = 3
key_buffer = 64M
sort_buffer = 64M
read_buffer = 16M
write_buffer = 16M

[mysqlhotcopy]
interactive-timeout
8. MySQL stats

+---------------------------------+------------------------------------------+
| Variable_name | Value |
+---------------------------------+------------------------------------------+
| back_log | 100 |
| basedir | /usr/local/mysql4/ |
| binlog_cache_size | 32768 |
| bulk_insert_buffer_size | 8388608 |
| character_set_client | latin1 |
| character_set_connection | latin1 |
| character_set_database | latin1 |
| character_set_results | latin1 |
| character_set_server | latin1 |
| character_set_system | utf8 |
| character_sets_dir | /usr/local/mysql4/share/mysql/charsets/ |
| collation_connection | latin1_swedish_ci |
| collation_database | latin1_swedish_ci |
| collation_server | latin1_swedish_ci |
| concurrent_insert | ON |
| connect_timeout | 5 |
| datadir | /var/lib/mysql/ |
| date_format | %Y-%m-%d |
| datetime_format | %Y-%m-%d %H:%i:%s |
| default_week_format | 0 |
| delay_key_write | ON |
| delayed_insert_limit | 100 |
| delayed_insert_timeout | 300 |
| delayed_queue_size | 1000 |
| expire_logs_days | 0 |
| flush | OFF |
| flush_time | 0 |
| ft_boolean_syntax | + -><()~*:""&| |
| ft_max_word_len | 84 |
| ft_min_word_len | 3 |
| ft_query_expansion_limit | 20 |
| ft_stopword_file | (built-in) |
| group_concat_max_len | 1024 |
| have_archive | YES |
| have_bdb | NO |
| have_blackhole_engine | NO |
| have_compress | YES |
| have_crypt | YES |
| have_csv | NO |
| have_example_engine | NO |
| have_geometry | YES |
| have_innodb | DISABLED |
| have_isam | NO |
| have_merge_engine | YES |
| have_ndbcluster | NO |
| have_openssl | NO |
| have_query_cache | YES |
| have_raid | YES |
| have_rtree_keys | YES |
| have_symlink | YES |
| init_connect | |
| init_file | |
| init_slave | |
| innodb_additional_mem_pool_size | 1048576 |
| innodb_autoextend_increment | 8 |
| innodb_buffer_pool_awe_mem_mb | 0 |
| innodb_buffer_pool_size | 8388608 |
| innodb_data_file_path | |
| innodb_data_home_dir | |
| innodb_fast_shutdown | ON |
| innodb_file_io_threads | 4 |
| innodb_file_per_table | OFF |
| innodb_flush_log_at_trx_commit | 1 |
| innodb_flush_method | |
| innodb_force_recovery | 0 |
| innodb_lock_wait_timeout | 50 |
| innodb_locks_unsafe_for_binlog | OFF |
| innodb_log_arch_dir | |
| innodb_log_archive | OFF |
| innodb_log_buffer_size | 1048576 |
| innodb_log_file_size | 5242880 |
| innodb_log_files_in_group | 2 |
| innodb_log_group_home_dir | |
| innodb_max_dirty_pages_pct | 90 |
| innodb_max_purge_lag | 0 |
| innodb_mirrored_log_groups | 1 |
| innodb_open_files | 300 |
| innodb_table_locks | ON |
| innodb_thread_concurrency | 8 |
| interactive_timeout | 28800 |
| join_buffer_size | 2093056 |
| key_buffer_size | 268435456 |
| key_cache_age_threshold | 300 |
| key_cache_block_size | 1024 |
| key_cache_division_limit | 100 |
| language | /usr/local/mysql4/share/mysql/english/ |
| large_files_support | ON |
| lc_time_names | en_US |
| license | GPL |
| local_infile | ON |
| locked_in_memory | OFF |
| log | OFF |
| log_bin | OFF |
| log_error | |
| log_slave_updates | OFF |
| log_slow_queries | ON |
| log_update | OFF |
| log_warnings | 1 |
| long_query_time | 5 |
| low_priority_updates | OFF |
| lower_case_file_system | OFF |
| lower_case_table_names | 0 |
| max_allowed_packet | 134216704 |
| max_binlog_cache_size | 4294967295 |
| max_binlog_size | 1073741824 |
| max_connect_errors | 10 |
| max_connections | 800 |
| max_delayed_threads | 20 |
| max_error_count | 64 |
| max_heap_table_size | 268434432 |
| max_insert_delayed_threads | 20 |
| max_join_size | 4294967295 |
| max_length_for_sort_data | 1024 |
| max_prepared_stmt_count | 16382 |
| max_relay_log_size | 0 |
| max_seeks_for_key | 4294967295 |
| max_sort_length | 1024 |
| max_tmp_tables | 32 |
| max_user_connections | 0 |
| max_write_lock_count | 4294967295 |
| myisam_data_pointer_size | 4 |
| myisam_max_extra_sort_file_size | 2147483648 |
| myisam_max_sort_file_size | 2147483647 |
| myisam_recover_options | OFF |
| myisam_repair_threads | 1 |
| myisam_sort_buffer_size | 67108864 |
| myisam_stats_method | nulls_unequal |
| net_buffer_length | 16384 |
| net_read_timeout | 30 |
| net_retry_count | 10 |
| net_write_timeout | 60 |
| new | OFF |
| old_passwords | ON |
| open_files_limit | 5810 |
| pid_file | /var/lib/mysql/db.realestniggas.com.pid |
| port | 3306 |
| preload_buffer_size | 32768 |
| prepared_stmt_count | 0 |
| protocol_version | 10 |
| query_alloc_block_size | 65536 |
| query_cache_limit | 8388608 |
| query_cache_min_res_unit | 4096 |
| query_cache_size | 134217728 |
| query_cache_type | ON |
| query_cache_wlock_invalidate | OFF |
| query_prealloc_size | 131072 |
| range_alloc_block_size | 2048 |
| read_buffer_size | 2093056 |
| read_only | OFF |
| read_rnd_buffer_size | 1044480 |
| relay_log_purge | ON |
| relay_log_space_limit | 0 |
| rpl_recovery_rank | 0 |
| secure_auth | OFF |
| server_id | 0 |
| skip_external_locking | ON |
| skip_networking | OFF |
| skip_show_database | OFF |
| slave_net_timeout | 3600 |
| slave_transaction_retries | 0 |
| slow_launch_time | 2 |
| socket | /var/lib/mysql/mysql.sock |
| sort_buffer_size | 4194296 |
| sql_mode | |
| sql_notes | OFF |
| sql_warnings | OFF |
| storage_engine | MyISAM |
| sync_binlog | 0 |
| sync_frm | ON |
| sync_replication | 0 |
| sync_replication_slave_id | 0 |
| sync_replication_timeout | 0 |
| system_time_zone | CST |
| table_cache | 2500 |
| table_type | MyISAM |
| thread_cache_size | 384 |
| thread_stack | 126976 |
| time_format | %H:%i:%s |
| time_zone | SYSTEM |
| tmp_table_size | 268435456 |
| tmpdir | |
| transaction_alloc_block_size | 8192 |
| transaction_prealloc_size | 4096 |
| tx_isolation | REPEATABLE-READ |
| version | 4.1.22-standard-log |
| version_comment | MySQL Community Edition - Standard (GPL) |
| version_compile_machine | i686 |
| version_compile_os | pc-linux-gnu |
| wait_timeout | 45 |
+---------------------------------+------------------------------------------+
+----------------------------+------------+
| Variable_name | Value |
+----------------------------+------------+
| Aborted_clients | 35 |
| Aborted_connects | 5 |
| Binlog_cache_disk_use | 0 |
| Binlog_cache_use | 0 |
| Bytes_received | 332463708 |
| Bytes_sent | 4221387965 |
| Com_admin_commands | 0 |
| Com_alter_db | 0 |
| Com_alter_table | 0 |
| Com_analyze | 0 |
| Com_backup_table | 0 |
| Com_begin | 0 |
| Com_change_db | 58695 |
| Com_change_master | 0 |
| Com_check | 0 |
| Com_checksum | 0 |
| Com_commit | 0 |
| Com_create_db | 0 |
| Com_create_function | 0 |
| Com_create_index | 0 |
| Com_create_table | 0 |
| Com_dealloc_sql | 0 |
| Com_delete | 6685 |
| Com_delete_multi | 0 |
| Com_do | 0 |
| Com_drop_db | 0 |
| Com_drop_function | 0 |
| Com_drop_index | 0 |
| Com_drop_table | 0 |
| Com_drop_user | 0 |
| Com_execute_sql | 0 |
| Com_flush | 0 |
| Com_grant | 0 |
| Com_ha_close | 0 |
| Com_ha_open | 0 |
| Com_ha_read | 0 |
| Com_help | 0 |
| Com_insert | 28251 |
| Com_insert_select | 88 |
| Com_kill | 0 |
| Com_load | 0 |
| Com_load_master_data | 0 |
| Com_load_master_table | 0 |
| Com_lock_tables | 175 |
| Com_optimize | 0 |
| Com_preload_keys | 0 |
| Com_prepare_sql | 0 |
| Com_purge | 0 |
| Com_purge_before_date | 0 |
| Com_rename_table | 0 |
| Com_repair | 0 |
| Com_replace | 4577 |
| Com_replace_select | 0 |
| Com_reset | 0 |
| Com_restore_table | 0 |
| Com_revoke | 0 |
| Com_revoke_all | 0 |
| Com_rollback | 0 |
| Com_savepoint | 0 |
| Com_select | 309361 |
| Com_set_option | 2 |
| Com_show_binlog_events | 0 |
| Com_show_binlogs | 0 |
| Com_show_charsets | 0 |
| Com_show_collations | 0 |
| Com_show_column_types | 0 |
| Com_show_create_db | 0 |
| Com_show_create_table | 0 |
| Com_show_databases | 0 |
| Com_show_errors | 0 |
| Com_show_fields | 0 |
| Com_show_grants | 0 |
| Com_show_innodb_status | 0 |
| Com_show_keys | 0 |
| Com_show_logs | 0 |
| Com_show_master_status | 0 |
| Com_show_ndb_status | 0 |
| Com_show_new_master | 0 |
| Com_show_open_tables | 0 |
| Com_show_privileges | 0 |
| Com_show_processlist | 1574 |
| Com_show_slave_hosts | 0 |
| Com_show_slave_status | 0 |
| Com_show_status | 19 |
| Com_show_storage_engines | 0 |
| Com_show_tables | 2 |
| Com_show_variables | 13 |
| Com_show_warnings | 0 |
| Com_slave_start | 0 |
| Com_slave_stop | 0 |
| Com_stmt_close | 0 |
| Com_stmt_execute | 0 |
| Com_stmt_prepare | 0 |
| Com_stmt_reset | 0 |
| Com_stmt_send_long_data | 0 |
| Com_truncate | 0 |
| Com_unlock_tables | 175 |
| Com_update | 88045 |
| Com_update_multi | 0 |
| Connections | 60284 |
| Created_tmp_disk_tables | 175 |
| Created_tmp_files | 370 |
| Created_tmp_tables | 24323 |
| Delayed_errors | 0 |
| Delayed_insert_threads | 0 |
| Delayed_writes | 0 |
| Flush_commands | 1 |
| Handler_commit | 0 |
| Handler_delete | 7385 |
| Handler_discover | 0 |
| Handler_read_first | 121218 |
| Handler_read_key | 16639512 |
| Handler_read_next | 112104117 |
| Handler_read_prev | 824802 |
| Handler_read_rnd | 316485 |
| Handler_read_rnd_next | 107879409 |
| Handler_rollback | 0 |
| Handler_update | 283956 |
| Handler_write | 61694 |
| Key_blocks_not_flushed | 0 |
| Key_blocks_unused | 0 |
| Key_blocks_used | 231960 |
| Key_read_requests | 72693046 |
| Key_reads | 539458 |
| Key_write_requests | 80507 |
| Key_writes | 64951 |
| Max_used_connections | 102 |
| Not_flushed_delayed_rows | 0 |
| Open_files | 575 |
| Open_streams | 0 |
| Open_tables | 454 |
| Opened_tables | 460 |
| Qcache_free_blocks | 4126 |
| Qcache_free_memory | 97709424 |
| Qcache_hits | 262040 |
| Qcache_inserts | 308261 |
| Qcache_lowmem_prunes | 0 |
| Qcache_not_cached | 925 |
| Qcache_queries_in_cache | 22458 |
| Qcache_total_blocks | 49455 |
| Questions | 819944 |
| Rpl_status | NULL |
| Select_full_join | 4 |
| Select_full_range_join | 0 |
| Select_range | 91269 |
| Select_range_check | 0 |
| Select_scan | 26392 |
| Slave_open_temp_tables | 0 |
| Slave_retried_transactions | 0 |
| Slave_running | OFF |
| Slow_launch_threads | 0 |
| Slow_queries | 27 |
| Sort_merge_passes | 188 |
| Sort_range | 72242 |
| Sort_rows | 18445212 |
| Sort_scan | 6298 |
| Table_locks_immediate | 875733 |
| Table_locks_waited | 4403 |
| Threads_cached | 96 |
| Threads_connected | 6 |
| Threads_created | 102 |
| Threads_running | 2 |
| Uptime | 7807 |
+----------------------------+------------+
Uptime: 7807 Threads: 6 Questions: 819945 Slow queries: 27 Opens: 460 Flush tables: 1 Open tables: 454 Queries per second avg: 105.027
mysqladmin Ver 8.41 Distrib 4.1.20, for redhat-linux-gnu on x86_64
Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
This software comes with ABSOLUTELY NO WARRANTY. This is free software,
and you are welcome to modify and redistribute it under the GPL license

Server version 4.1.22-standard-log
Protocol version 10
Connection 192.168.1.3 via TCP/IP
TCP port 3306
Uptime: 2 hours 10 min 7 sec

Threads: 6 Questions: 819945 Slow queries: 27 Opens: 460 Flush tables: 1 Open tables: 454 Queries per second avg: 105.027
+-------+---------------+-------------------+-----------------------+---------+------+----------------+-------------------------------------------------------
-----------------------------------------------+
| Id | User | Host | db | Command | Time | State | Info
|
+-------+---------------+-------------------+-----------------------+---------+------+----------------+-------------------------------------------------------
-----------------------------------------------+
| 60263 | realestn_vbul | 192.168.1.2:51161 | realestn_realestnvbul | Sleep | 1 | |
|
| 60279 | realestn_vbul | 192.168.1.2:51177 | realestn_realestnvbul | Sleep | 1 | |
|
| 60280 | realestn_vbul | 192.168.1.2:51178 | realestn_realestnvbul | Query | 0 | Sorting result | SELECT userid, dateline
FROM reputation
WHERE whoadded = 21218
ORDER BY dateline DESC
|
| 60281 | realestn_vbul | 192.168.1.2:51179 | realestn_realestnvbul | Sleep | 0 | |
|
| 60282 | realestn_vbul | 192.168.1.2:51180 | realestn_realestnvbul | Sleep | 0 | |
|
| 60283 | root | 192.168.1.2:51181 | | Query | 0 | | show processlist
|
+-------+---------------+-------------------+-----------------------+---------+------+----------------+-------------------------------------------------------
-----------------------------------------------+
MySQL 4.1.22-standard-l uptime 0 2:13:56 Mon Feb 11 23:58:14 2008

__ Key __________________________________________________ _______________
Buffer used 226.52M of 256.00M %Used: 88.49
Current 256.00M %Usage: 100.00
Write hit 19.05%
Read hit 99.27%

__ Questions __________________________________________________ _________
Total 839.23k 104.4/s
DMS 447.28k 55.7/s %Total: 53.30
QC Hits 268.25k 33.4/s 31.96
Com_ 62.05k 7.7/s 7.39
COM_QUIT 61.66k 7.7/s 7.35
-Unknown 20 0.0/s 0.00
Slow 28 0.0/s 0.00 %DMS: 0.01
DMS 447.28k 55.7/s 53.30
SELECT 316.61k 39.4/s 37.73 70.78
UPDATE 90.09k 11.2/s 10.73 20.14
INSERT 29.04k 3.6/s 3.46 6.49
DELETE 6.88k 0.9/s 0.82 1.54
REPLACE 4.67k 0.6/s 0.56 1.04
Com_ 62.05k 7.7/s 7.39
change_db 60.09k 7.5/s 7.16
show_proces 1.57k 0.2/s 0.19
lock_tables 175 0.0/s 0.02

__ SELECT and Sort __________________________________________________ ___
Scan 27.02k 3.4/s %SELECT: 8.54
Range 93.28k 11.6/s 29.46
Full join 4 0.0/s 0.00
Range check 0 0/s 0.00
Full rng join 0 0/s 0.00
Sort scan 6.45k 0.8/s
Sort range 73.93k 9.2/s
Sort mrg pass 192 0.0/s

__ Query Cache __________________________________________________ _______
Memory usage 33.31M of 128.00M %Used: 26.03
Block Fragmnt 8.56%
Hits 268.25k 33.4/s
Inserts 315.49k 39.3/s
Insrt:Prune 315.49k:1 39.3/s
Hit:Insert 0.85:1

__ Table Locks __________________________________________________ _______
Waited 4.46k 0.6/s %Total: 0.50
Immediate 896.28k 111.5/s

__ Tables __________________________________________________ ____________
Open 454 of 2500 %Cache: 18.16
Opened 460 0.1/s

__ Connections __________________________________________________ _______
Max used 102 of 800 %Max: 12.75
Total 61.68k 7.7/s

__ Created Temp __________________________________________________ ______
Disk table 179 0.0/s
Table 24.84k 3.1/s
File 378 0.0/s

__ Threads __________________________________________________ ___________
Running 1 of 2
Cached 100 of 384 %Hit: 99.83
Created 102 0.0/s
Slow 0 0/s

__ Aborted __________________________________________________ ___________
Clients 35 0.0/s
Connects 5 0.0/s

__ Bytes __________________________________________________ _____________
Sent 435.57M 54.2k/s
Received 340.49M 42.4k/s
9. Only vbulletin

10. Not sure

11. http://72.232.84.132/phpinfo.php

12. httpd.conf

Timeout 100
KeepAlive Off
MaxKeepAliveRequests 100
KeepAliveTimeout 1

# prefork MPM
# StartServers: number of server processes to start
# MinSpareServers: minimum number of server processes which are kept spare
# MaxSpareServers: maximum number of server processes which are kept spare
# MaxClients: maximum number of server processes allowed to start
# MaxRequestsPerChild: maximum number of requests a server process serves

<IfModule prefork.c>
MaxClients 170
StartServers 15
MinSpareServers 15
MaxSpareServers 20
#ServerLimit 1048
MaxRequestsPerChild 1000
</IfModule>

<IfModule worker.c>
StartServers 15
MaxClients 150
MinSpareThreads 25
MaxSpareThreads 75
ThreadsPerChild 25
MaxRequestsPerChild 1000
</IfModule>

# perchild MPM
# NumServers: constant number of server processes
# StartThreads: initial number of worker threads in each server process
# MinSpareThreads: minimum number of worker threads which are kept spare
# MaxSpareThreads: maximum number of worker threads which are kept spare
# MaxThreadsPerChild: maximum number of worker threads in each server process
# MaxRequestsPerChild: maximum number of connections per server process
<IfModule perchild.c>
NumServers 15
StartThreads 15
MinSpareThreads 15
MaxSpareThreads 20
MaxThreadsPerChild 20
MaxRequestsPerChild 1000
</IfModule>
13. No big files

14.

uname -a
Linux www.realestniggas.com 2.6.9-55.ELsmp #1 SMP Wed May 2 14:04:42 EDT 2007 x86_64 x86_64 x86_64 GNU/Linux

ulimit -aH
core file size (blocks, -c) unlimited
data seg size (kbytes, -d) unlimited
file size (blocks, -f) unlimited
pending signals (-i) 1024
max locked memory (kbytes, -l) 32
max memory size (kbytes, -m) unlimited
open files (-n) 1024
pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200
stack size (kbytes, -s) unlimited
cpu time (seconds, -t) unlimited
max user processes (-u) 40960
virtual memory (kbytes, -v) unlimited
file locks (-x) unlimited

/proc/cpuinfo
processor : 0
vendor_id : AuthenticAMD
cpu family : 15
model : 37
model name : AMD Opteron(tm) Processor 254
stepping : 1
cpu MHz : 2813.024
cache size : 1024 KB
fpu : yes
fpu_exception : yes
cpuid level : 1
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 syscall nx mmxext lm 3dnowext 3dnow pni
bogomips : 5631.62
TLB size : 1088 4K pages
clflush size : 64
cache_alignment : 64
address sizes : 40 bits physical, 48 bits virtual
power management: ts fid vid ttp

processor : 1
vendor_id : AuthenticAMD
cpu family : 15
model : 37
model name : AMD Opteron(tm) Processor 254
stepping : 1
cpu MHz : 2813.024
cache size : 1024 KB
fpu : yes
fpu_exception : yes
cpuid level : 1
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 syscall nx mmxext lm 3dnowext 3dnow pni
bogomips : 5625.12
TLB size : 1088 4K pages
clflush size : 64
cache_alignment : 64
address sizes : 40 bits physical, 48 bits virtual
power management: ts fid vid ttp
15. slow query

__ 001 __________________________________________________ _____________________

Count : 286 (61%)
Time (seconds): 2760 total, 9.65 avg, 6 to 32 max
95% of Time : 2434 total, 8.98 avg, 6 to 16 max
Lock (seconds): 30 total, 0.10 avg, 0 to 13 max
Rows sent : 120126 avg, 5154 to 1767111 max
Rows examined : 120126 avg, 5154 to 1767111 max
User : realestn_vbul[realestn_vbul]@/192.168.1.2 (100%)
Database : realestn_realestnvbul

SELECT postid, intitle FROM postindex WHERE wordid IN (N1);

__ 002 __________________________________________________ _____________________

Count : 35 (7%)
Time (seconds): 482 total, 13.77 avg, 6 to 50 max
95% of Time : 397 total, 12.03 avg, 6 to 29 max
Lock (seconds): 0 total, 0 avg, 0 to 0 max
Rows sent : 0 avg, 0 to 13 max
Rows examined : 0 avg, 0 to 13 max
User : realestn_vbul[realestn_vbul]@/192.168.1.2 (100%)
Database : Unknown

SELECT * FROM datastore WHERE title IN (S14);

__ 003 __________________________________________________ _____________________

Count : 38 (8%)
Time (seconds): 430 total, 11.32 avg, 6 to 94 max
95% of Time : 309 total, 8.58 avg, 6 to 21 max
Lock (seconds): 2 total, 0.05 avg, 0 to 2 max
Rows sent : 1339048 avg, 0 to 1767401 max
Rows examined : 1339048 avg, 0 to 1767401 max
User : realestn_vbul[realestn_vbul]@/192.168.1.2 (100%)
Database : Unknown

SELECT postid FROM postindex WHERE wordid IN (N1);

__ 004 __________________________________________________ _____________________

Count : 26 (5%)
Time (seconds): 363 total, 13.96 avg, 6 to 60 max
95% of Time : 244 total, 10.17 avg, 6 to 20 max
Lock (seconds): 0 total, 0 avg, 0 to 0 max
Rows sent : 0 avg, 0 to 0 max
Rows examined : 0 avg, 0 to 0 max
User : realestn_vbul[realestn_vbul]@/192.168.1.2 (100%)
Database : Unknown

SELECT * FROM datastore WHERE title IN (S15);

__ 005 __________________________________________________ _____________________

Count : 11 (2%)
Time (seconds): 244 total, 22.18 avg, 6 to 110 max
95% of Time : 134 total, 13.40 avg, 6 to 31 max
Lock (seconds): 0 total, 0 avg, 0 to 0 max
Rows sent : 0 avg, 0 to 0 max
Rows examined : 0 avg, 0 to 0 max
User : realestn_vbul[realestn_vbul]@/192.168.1.2 (100%)
Database : Unknown

SELECT * FROM datastore WHERE title IN (S18);

__ 006 __________________________________________________ _____________________

Count : 9 (1%)
Time (seconds): 112 total, 12.44 avg, 6 to 35 max
95% of Time : 77 total, 9.62 avg, 6 to 16 max
Lock (seconds): 0 total, 0 avg, 0 to 0 max
Rows sent : 1 avg, 0 to 12 max
Rows examined : 1 avg, 0 to 12 max
User : realestn_vbul[realestn_vbul]@/192.168.1.2 (100%)
Database : Unknown

SELECT * FROM datastore WHERE title IN (S13);

__ 007 __________________________________________________ _____________________

Count : 4 (0%)
Time (seconds): 85 total, 21.25 avg, 18 to 23 max
95% of Time : 62 total, 20.67 avg, 18 to 22 max
Lock (seconds): 0 total, 0 avg, 0 to 0 max
Rows sent : 1 avg, 1 to 1 max
Rows examined : 15011323 avg, 15000701 to 15016436 max
User : realestn_vbul[realestn_vbul]@/192.168.1.2 (100%)
Database : Unknown

SELECT COUNT(*) AS count FROM post WHERE dateline >= N;

__ 008 __________________________________________________ _____________________

Count : 2 (0%)
Time (seconds): 61 total, 30.50 avg, 25 to 36 max
95% of Time : 25 total, 25.00 avg, 25 to 25 max
Lock (seconds): 0 total, 0 avg, 0 to 0 max
Rows sent : 0 avg, 0 to 0 max
Rows examined : 18086 avg, 18086 to 18086 max
User : realestn_vbul[realestn_vbul]@/192.168.1.2 (100%)
Database : Unknown

SELECT wordid, title FROM word WHERE title like('S');

__ 009 __________________________________________________ _____________________

Count : 4 (0%)
Time (seconds): 45 total, 11.25 avg, 9 to 18 max
95% of Time : 27 total, 9.00 avg, 9 to 9 max
Lock (seconds): 8 total, 2 avg, 0 to 8 max
Rows sent : 3 avg, 3 to 3 max
Rows examined : 1958294 avg, 1957883 to 1959527 max
User : realestn_vbul[realestn_vbul]@/192.168.1.2 (100%)
Database : Unknown

SELECT item.item_id, item.item_title, item.threadid, grp.group_title, item.item_status, item.item_created, item.item_open_until, item.item_pay_after, thread.threadid, thread.forumid, thread.postuserid FROM vbookie_items AS item LEFT JOIN vbookie_groups AS grp ON grp.group_id = item.group_id LEFT JOIN thread AS thread ON thread.vbookie_item_id = item.item_id WHERE item.item_status='S' ORDER BY item.item_open_until, item.item_pay_after, item.item_id DESC LIMIT N, N;

__ 010 __________________________________________________ _____________________

Count : 3 (0%)
Time (seconds): 41 total, 13.67 avg, 11 to 18 max
95% of Time : 23 total, 11.50 avg, 11 to 12 max
Lock (seconds): 0 total, 0 avg, 0 to 0 max
Rows sent : 140881 avg, 62199 to 232493 max
Rows examined : 140881 avg, 62199 to 232493 max
User : realestn_vbul[realestn_vbul]@/192.168.1.2 (100%)
Database : Unknown

SELECT postid, score AS origscore, case intitle when N then score + N when N then score + N else score end AS score FROM postindex WHERE wordid IN (N1);

eva2000
Wed 13th Feb '08, 5:07am
Maxclients increases just means you're concurrent traffic is increasing so you'd need to raise the limit

Try the following in this exact order. You can ignore any of the suggestions that you have already done.

1. Upgrade MySQL server to 4.1.22
2. Upgrade PHP to 5.2.5 if you're on 5.x or PHP 4.4.8 if you're on PHP 4.x after upgrading MySQL (ensure you install as ISAPI and NOT CGI)
3. Edit mysql server's /etc/my.cnf or c:\my.ini for windows and place the following mysql server settings in /etc/my.cnf and restart mysql server afterwards. Make sure to restart mysql server everytime you make changes to your my.cnf for the changes to take effect.

If mysql doesn't restart properly after my.cnf changes and you're on VPS server, make sure skip-innodb entry is removed or commented out from below my.cnf


[mysqld]
ft_min_word_len = 3
datadir = /var/lib/mysql
safe-show-database
old_passwords
back_log = 75
skip-innodb
max_connections = 500
key_buffer = 384M
myisam_sort_buffer_size = 128M
join_buffer_size = 1M
read_buffer_size = 1M
sort_buffer_size = 2M
table_cache = 4000
thread_cache_size = 512
wait_timeout = 15
connect_timeout = 10
tmp_table_size = 128M
max_heap_table_size = 128M
max_allowed_packet = 64M
net_buffer_length = 16384
max_connect_errors = 10
thread_concurrency = 4
read_rnd_buffer_size = 786432
bulk_insert_buffer_size = 8M
query_cache_limit = 4M
query_cache_size = 64M
query_cache_type = 1
query_prealloc_size = 262144
query_alloc_block_size = 65536
transaction_alloc_block_size = 8192
transaction_prealloc_size = 4096
default-storage-engine = MyISAM

[mysql.server]
user = mysql
basedir = /var/lib

[mysqld_safe]
log-error = /var/log/mysqld-err.log
pid-file = /var/lib/mysql/mysql.pid
socket = /var/lib/mysql/mysql.sock
nice = -10
open_files_limit = 8192

[mysqldump]
quick
max_allowed_packet = 16M

[myisamchk]
ft_min_word_len = 3
key_buffer = 64M
sort_buffer = 64M
read_buffer = 16M
write_buffer = 16M

[mysqlhotcopy]
interactive-timeout


If you get mysql server gone away error messages, then keep increasing wait_timeout value in my.cnf by 60 second increments, then restart mysql after my.cnf changes for it to take effect. Test for a few days and see if you get less or eliminate that error message. If it still occurs, then keep repeating the 60 second increment until the message goes away. Each vB forum and server will have different optimal wait_timeout values depending on your vB forum traffic patterns and server hardware specifications.

4. Remove Eaccelerator or APC Cache if they're installed (check phpinfo.php url of yours to see) and instead install Xcache v1.2.1 final release or Xcache v1.2.2 http://xcache.lighttpd.net/wiki/Release-1.2.2 which seems to be a bit faster than APC Cache - http://www.vbulletin.com/forum/showthread.php?t=213267. Xcache site http://xcache.lighttpd.net/, documentation http://trac.lighttpd.net/trac/wiki/Docs and forums http://forum.lighttpd.net/forum/4

5. Upgrade to vB 3.0.17 http://www.vbulletin.com/forum/showthread.php?t=209720 if you're on vB 3.0.xx or upgrade to vB 3.5.8 http://www.vbulletin.com/forum/showthread.php?t=221903 if you're on vB 3.5.x. Upgrade to vB 3.6.8 PL2 http://www.vbulletin.com/forum/showthread.php?t=247739 if you're on vB 3.6.x

6. If you just upgraded to vB 3.5.x/3.6.x try to disable these 4 options:

Admin CP -> vBulletin Options -> Forums Home Page Options -> Display Logged in Users?

Admin CP -> vBulletin Options -> Forum Display Options (forumdisplay) -> Show Users Browsing Forums

Admin CP -> vBulletin Options -> Thread Display Options -> Show Users Browsing Thread

Admin CP -> vBulletin Options -> Message Searching Options -> Automatic Similar Thread search

7. Check phpinfo.php url to see if you have mod_gzip (Apache 1.3.x) or mod_deflate (Apache 2.x) loaded/installed (on phpinfo.php url page in browser you can do CTRL+F to bring up find prompt and type in mod_gzip or mod_deflate to quickly see). If you have either mod_gzip or mod_deflate loaded, then ensure vB admincp gzip compression is disabled since it's the same thing as mod_gzip or mod_deflate and double compression will just increase cpu loads. If you don't have mod_gzip or mod_deflate installed, then DO NOT set vB admincp gzip compression to a level higher than 1. Higher than 1, will only increase cpu loads unncessarily.

8. Edit httpd.conf values to following and restart apache

Timeout 100
KeepAlive Off or On (try both and see which handles better)
MaxKeepAliveRequests 100
KeepAliveTimeout 1

<IfModule prefork.c>
MaxClients 256
StartServers 15
MinSpareServers 15
MaxSpareServers 20
MaxRequestsPerChild 1000
</IfModule>

9. You're using mysql 3.23.49 client libraries according to PHPINFO, so you may need to install same mysql version on web server, and recompile PHP with the --with-mysql=/path/to/mysql option to tell PHP to use the latest mysql 4.1.x client libraries i.e. if mysqldump/mysqlcheck/mysqladmin are located at /usr/bin/ or /usr/local/mysql/bin then you need to compile PHP with --with-mysql=/usr or --with-mysql=/usr/local/mysql - just had a few issues with one vB client have issues with PHP connecting to external mysql server until i did this.

10. If you have split web + db servers, ensure web server has dual network cards as outlined at http://www.vbulletin.org/forum/showthread.php?t=111191

11. Also might want to use Xcache datastore hack on vbulletin.org for vB 3.6.x to save on queries to the datastore table. vB 3.7.x has Xcache datastore support officially via config.php

rixey
Thu 14th Feb '08, 11:32am
1. Upgrade MySQL server to 4.1.22
MySQL is now at version 6. Would it be better to use version 5 or 6 instead? or version 4.1.22 is the most stable and compatible with vbulletin. I'd like to know your reason on suggesting to use mysql 4.


Maxclient 256

I've tried this and when it does reach the maxclient, the web server became unaccessible due to heavy load.

Thanks for the help, I'll update you with the result.

eva2000
Fri 15th Feb '08, 7:38am
stick with mysql 4.1.22 or 5.0.51.. version 6 AFAIK hasn't been fully tested by our vbulletin developers as yet.

rixey
Thu 21st Feb '08, 9:54pm
Update: I didn't set maxclient to 256 but instead increased them gradually. I enabled the persistent connection and seems to be solving the maxclient issue.

Right now, I'm having issue with the server keep timing out and slow to upload. Here's what the users complained

1. sometimes image load and sometimes dont
2. sometimes not all
3. sometimes takes more then 5 mins to load.

i was wondering if its because of insufficient memory.


WEB Server

# free -m
total used free shared buffers cached
Mem: 3943 3098 844 0 37 92
-/+ buffers/cache: 2969 973
== edit start ==

Enabled KeepAlive and for the past two hours seems to be good too. Memory usage looks reasonable.


total used free shared buffers cached
Mem: 3943 1233 2709 0 37 94
-/+ buffers/cache: 1101 2841
Swap: 1992 8 1984

== edit end ==

Thanks eva

eva2000
Fri 22nd Feb '08, 5:20am
in both cases memory looks fine to me.

Tried bumping maxclients to say 300 ?

rixey
Fri 22nd Feb '08, 5:26am
The first "free -m" i left out the swap memory where the server already used some of it (more then 8MB). That would explain the slowness and timeout.

Currently its keepalive, pconnect on and maxclient 180. seems to be working good enough. We passed through the peak hour without any problem today but previously it had problem after a day or two.

i tried with maxclient 256 but the server became unresponsive due the high load 100++.

eva2000
Fri 22nd Feb '08, 8:10am
swap usage of 8M is nothing to worry about unless you're hard drives are very very SLOW !

rixey
Fri 22nd Feb '08, 1:35pm
yeah i know 8MB is nothing to worry but it went more then that. Well, I'll just see what happen for new few days. Will be updating if there's anything wrong. Thank you so much Eva

rixey
Tue 26th Feb '08, 10:50am
Server died several time due to out of memory killer. Not sure if this would help but here it goes.


Feb 26 08:48:30 www kernel: oom-killer: gfp_mask=0xd0
Feb 26 08:48:30 www kernel: Mem-info:
Feb 26 08:48:30 www kernel: Node 1 DMA per-cpu: empty
Feb 26 08:48:30 www kernel: Node 1 Normal per-cpu:
Feb 26 08:48:30 www kernel: cpu 0 hot: low 32, high 96, batch 16
Feb 26 08:48:30 www kernel: cpu 0 cold: low 0, high 32, batch 16
Feb 26 08:48:30 www kernel: cpu 1 hot: low 32, high 96, batch 16
Feb 26 08:48:30 www kernel: cpu 1 cold: low 0, high 32, batch 16
Feb 26 08:48:30 www kernel: Node 1 HighMem per-cpu: empty
Feb 26 08:48:30 www kernel: Node 0 DMA per-cpu:
Feb 26 08:48:32 www kernel: cpu 0 hot: low 2, high 6, batch 1
Feb 26 08:48:33 www kernel: cpu 0 cold: low 0, high 2, batch 1
Feb 26 08:48:33 www kernel: cpu 1 hot: low 2, high 6, batch 1
Feb 26 08:48:33 www kernel: cpu 1 cold: low 0, high 2, batch 1
Feb 26 08:48:33 www kernel: Node 0 Normal per-cpu:
Feb 26 08:48:33 www kernel: cpu 0 hot: low 32, high 96, batch 16
Feb 26 08:48:33 www kernel: cpu 0 cold: low 0, high 32, batch 16
Feb 26 08:48:33 www kernel: cpu 1 hot: low 32, high 96, batch 16
Feb 26 08:48:33 www kernel: cpu 1 cold: low 0, high 32, batch 16
Feb 26 08:48:33 www kernel: Node 0 HighMem per-cpu: empty
Feb 26 08:48:33 www kernel:
Feb 26 08:48:33 www kernel: Free pages: 20764kB (0kB HighMem)
Feb 26 08:48:33 www kernel: Active:502340 inactive:475589 dirty:0 writeback:0 unstable:0 free:5191 slab:8588 mapped:981568 pagetables:7605
Feb 26 08:48:33 www kernel: Node 1 DMA free:0kB min:0kB low:0kB high:0kB active:0kB inactive:0kB present:0kB pages_scanned:0 all_unreclaimable? no
Feb 26 08:48:33 www kernel: protections[]: 0 0 0
Feb 26 08:48:33 www kernel: Node 1 Normal free:5440kB min:5484kB low:10968kB high:16452kB active:104148kB inactive:1873444kB present:3145724kB pages_scanned:3
365571 all_unreclaimable? yes
Feb 26 08:48:34 www kernel: protections[]: 0 0 0
Feb 26 08:48:34 www kernel: Node 1 HighMem free:0kB min:128kB low:256kB high:384kB active:0kB inactive:0kB present:0kB pages_scanned:0 all_unreclaimable? no
Feb 26 08:48:34 www kernel: protections[]: 0 0 0
Feb 26 08:48:34 www kernel: Node 0 DMA free:11820kB min:28kB low:56kB high:84kB active:0kB inactive:0kB present:16384kB pages_scanned:0 all_unreclaimable? yes
Feb 26 08:48:34 www kernel: protections[]: 0 0 0
Feb 26 08:48:34 www kernel: Node 0 Normal free:3504kB min:3628kB low:7256kB high:10884kB active:1905340kB inactive:28784kB present:2080764kB pages_scanned:824
1913 all_unreclaimable? yes
Feb 26 08:48:35 www kernel: protections[]: 0 0 0
Feb 26 08:48:35 www kernel: Node 0 HighMem free:0kB min:128kB low:256kB high:384kB active:0kB inactive:0kB present:0kB pages_scanned:0 all_unreclaimable? no
Feb 26 08:48:35 www kernel: protections[]: 0 0 0
Feb 26 08:48:35 www kernel: Node 1 DMA: empty
Feb 26 08:48:35 www kernel: Node 1 Normal: 0*4kB 0*8kB 0*16kB 8*32kB 1*64kB 0*128kB 0*256kB 0*512kB 1*1024kB 0*2048kB 1*4096kB = 5440kB
Feb 26 08:48:35 www kernel: Node 1 HighMem: empty
Feb 26 08:48:36 www kernel: Node 0 DMA: 5*4kB 5*8kB 1*16kB 3*32kB 2*64kB 2*128kB 2*256kB 1*512kB 0*1024kB 1*2048kB 2*4096kB = 11820kB
Feb 26 08:48:36 www kernel: Node 0 Normal: 0*4kB 0*8kB 1*16kB 1*32kB 2*64kB 0*128kB 1*256kB 0*512kB 1*1024kB 1*2048kB 0*4096kB = 3504kB
Feb 26 08:48:37 www kernel: Node 0 HighMem: empty
Feb 26 08:48:37 www kernel: Swap cache: add 2695476, delete 2695476, find 14565907/14675180, race 33+48
Feb 26 08:48:37 www kernel: Free swap: 0kB
Feb 26 08:48:37 www kernel: 1310718 pages of RAM
Feb 26 08:48:38 www kernel: 301308 reserved pages
Feb 26 08:48:38 www kernel: 29899 pages shared
Feb 26 08:48:38 www kernel: 0 pages swap cached
Feb 26 08:48:38 www kernel: Out of Memory: Killed process 16354 (httpd).

rixey
Wed 27th Feb '08, 7:19am
I'm saw this on top, it shows that one of httpd process is using 17.3% memory.

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
10658 apache 16 0 754m 682m 2416 S 0.0 17.3 0:31.45 /usr/local/apache-2.2.8//bin/httpd -k start


[root@www ~]# strace -p 10658
Process 10658 attached - interrupt to quit
futex(0x3b68e31640, FUTEX_WAIT, 2, NULL

eva2000
Thu 28th Feb '08, 8:25am
tried updating apache as well ?

rixey
Thu 6th Mar '08, 2:20am
Finally I can sleep better at night. Thank you so much eva for all the help!

As for the last few post, it was due to apache memory leak. I've reduced the MaxRequestPerChild to 500 to avoid this. So in case we're getting more visitors, I'll have to increase that and MaxClients.

Thanks again eva2000, you're the best there is.

eva2000
Thu 6th Mar '08, 7:17am
Glad you got it sorted out and thanks for letting us know what the issue was :)