PDA

View Full Version : Optimization request


Soyuz
Tue 25th Apr '06, 6:34am
1.- Dedicated server with only 1 web portal hosted (postnuke and vbulletin)

2.- Dual Xeon 2.8 Ghz, 4 Gb RAM, 2 x HD SCSI 73 Gb.
Debian, Linux 2.4.25-bf2.4
PHP 4.3.10-2
MySQL 4.0.23
Apache/1.3.33

3.- Yes, I use some INNODB tables

4.- I don't know

5.-
top - 11:02:53 up 4 days, 8:01, 1 user, load average: 4.67, 6.17, 7.39
Tasks: 204 total, 2 running, 202 sleeping, 0 stopped, 0 zombie
Cpu(s): 66.1% user, 12.8% system, 0.0% nice, 21.1% idle
Mem: 4076684k total, 3615932k used, 460752k free, 146712k buffers
Swap: 1999988k total, 38084k used, 1961904k free, 2000796k cached

6.-
# The following options will be passed to all MySQL clients
[client]
#password = your_password
port = 3306
socket = /var/run/mysqld/mysqld.sock

# Here follows entries for some specific programs

# The MySQL server
[mysqld]
#innodb_force_recovery = 3
port = 3306
socket = /var/run/mysqld/mysqld.sock
skip-locking
max_connections = 500
key_buffer = 128M
myisam_sort_buffer_size = 64M
join_buffer_size = 1M
read_buffer_size = 1M
sort_buffer_size = 2M
table_cache = 1800
thread_cache_size = 384
wait_timeout = 45
connect_timeout = 10
tmp_table_size = 256M
max_heap_table_size = 64M
max_allowed_packet = 64M
max_connect_errors = 10000
read_rnd_buffer_size = 524288
bulk_insert_buffer_size = 16M
query_cache_limit = 6M
query_cache_size = 160M
query_cache_type = 1
query_prealloc_size = 163840
query_alloc_block_size = 32768

# Try number of CPU's*2 for thread_concurrency
thread_concurrency = 4

#
skip-networking

# Uncomment the following if you are using InnoDB tables
innodb_data_home_dir = /var/lib/mysql/
innodb_data_file_path = ibdata1:10M:autoextend
innodb_log_group_home_dir = /var/lib/mysql/
innodb_log_arch_dir = /var/lib/mysql/
# You can set .._buffer_pool_size up to 50 - 80 %
# of RAM but beware of setting memory usage too high
innodb_buffer_pool_size = 512M
innodb_additional_mem_pool_size = 48M
# Set .._log_file_size to 25 % of buffer pool size
innodb_log_file_size = 64M
innodb_log_buffer_size = 8M
innodb_flush_log_at_trx_commit = 1
innodb_lock_wait_timeout = 50

language = /usr/share/mysql/spanish

[mysqld_safe]
open_files_limit = 8192

[mysqldump]
quick
max_allowed_packet = 16M

[mysql]
no-auto-rehash
# Remove the next comment character if you are not familiar with SQL
#safe-updates

[isamchk]
key_buffer = 128M
sort_buffer_size = 128M
read_buffer = 16M
write_buffer = 16M

[myisamchk]
key_buffer = 64M
sort_buffer = 64M
read_buffer = 16M
write_buffer = 16M

[mysqlhotcopy]
interactive-timeout


7.-

+---------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Variable_name | Value |
+---------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| back_log | 50 |
| basedir | /usr/ |
| bdb_cache_size | 8388600 |
| bdb_log_buffer_size | 921600 |
| bdb_home | /var/lib/mysql/ |
| bdb_max_lock | 10000 |
| bdb_logdir | |
| bdb_shared_data | OFF |
| bdb_tmpdir | /tmp/ |
| bdb_version | Sleepycat Software: Berkeley DB 3.2.9a: (December 18, 2004) |
| binlog_cache_size | 32768 |
| bulk_insert_buffer_size | 16777216 |
| character_set | latin1 |
| character_sets | latin1 big5 cp1251 cp1257 croat czech danish dec8 dos estonia euc_kr gb2312 gbk german1 greek hebrew hp8 hungarian koi8_ru koi8_ukr latin1_de latin2 latin5 sjis swe7 tis620 ujis usa7 win1250 win1251ukr win1251 |
| concurrent_insert | ON |
| connect_timeout | 10 |
| convert_character_set | |
| datadir | /var/lib/mysql/ |
| default_week_format | 0 |
| delay_key_write | ALL |
| delayed_insert_limit | 100 |
| delayed_insert_timeout | 300 |
| delayed_queue_size | 1000 |
| flush | OFF |
| flush_time | 0 |
| ft_boolean_syntax | + -><()~*:""&| |
| ft_min_word_len | 4 |
| ft_max_word_len | 254 |
| ft_max_word_len_for_sort | 20 |
| ft_stopword_file | (built-in) |
| have_bdb | YES |
| have_crypt | YES |
| have_innodb | YES |
| have_isam | YES |
| have_raid | YES |
| have_symlink | YES |
| have_openssl | NO |
| have_query_cache | YES |
| init_file | |
| innodb_additional_mem_pool_size | 50331648 |
| innodb_buffer_pool_size | 536870912 |
| innodb_data_file_path | ibdata1:10M:autoextend |
| innodb_data_home_dir | /var/lib/mysql/ |
| innodb_file_io_threads | 4 |
| innodb_force_recovery | 0 |
| innodb_thread_concurrency | 8 |
| innodb_flush_log_at_trx_commit | 1 |
| innodb_fast_shutdown | ON |
| innodb_flush_method | |
| innodb_lock_wait_timeout | 50 |
| innodb_log_arch_dir | /var/lib/mysql/ |
| innodb_log_archive | OFF |
| innodb_log_buffer_size | 8388608 |
| innodb_log_file_size | 67108864 |
| innodb_log_files_in_group | 2 |
| innodb_log_group_home_dir | /var/lib/mysql/ |
| innodb_mirrored_log_groups | 1 |
| innodb_max_dirty_pages_pct | 90 |
| innodb_max_purge_lag | 0 |
| innodb_table_locks | ON |
| interactive_timeout | 28800 |
| join_buffer_size | 1044480 |
| key_buffer_size | 134217728 |
| language | /usr/share/mysql/spanish/ |
| large_files_support | ON |
| license | GPL |
| local_infile | ON |
| locked_in_memory | OFF |
| log | OFF |
| log_update | OFF |
| log_bin | OFF |
| log_slave_updates | OFF |
| log_slow_queries | ON |
| log_warnings | 1 |
| long_query_time | 10 |
| low_priority_updates | OFF |
| lower_case_file_system | OFF |
| lower_case_table_names | 0 |
| max_allowed_packet | 67107840 |
| max_binlog_cache_size | 4294967295 |
| max_binlog_size | 1073741824 |
| max_connections | 500 |
| max_connect_errors | 10000 |
| max_delayed_threads | 20 |
| max_insert_delayed_threads | 20 |
| max_heap_table_size | 67107840 |
| max_join_size | 4294967295 |
| max_relay_log_size | 0 |
| max_seeks_for_key | 4294967295 |
| max_sort_length | 1024 |
| max_user_connections | 0 |
| max_tmp_tables | 32 |
| max_write_lock_count | 4294967295 |
| myisam_max_extra_sort_file_size | 268435456 |
| myisam_max_sort_file_size | 2147483647 |
| myisam_repair_threads | 1 |
| myisam_recover_options | OFF |
| myisam_sort_buffer_size | 67108864 |
| net_buffer_length | 16384 |
| net_read_timeout | 30 |
| net_retry_count | 10 |
| net_write_timeout | 60 |
| new | OFF |
| open_files_limit | 4110 |
| pid_file | /var/lib/mysql/server.hispasonic.com.pid |
| log_error | |
| port | 3306 |
| protocol_version | 10 |
| query_alloc_block_size | 32768 |
| query_cache_limit | 6291456 |
| query_cache_size | 167772160 |
| query_cache_type | ON |
| query_prealloc_size | 163840 |
| range_alloc_block_size | 2048 |
| read_buffer_size | 1044480 |
| read_only | OFF |
| read_rnd_buffer_size | 520192 |
| rpl_recovery_rank | 0 |
| server_id | 0 |
| slave_net_timeout | 3600 |
| skip_external_locking | ON |
| skip_networking | ON |
| skip_show_database | OFF |
| slow_launch_time | 2 |
| socket | /var/run/mysqld/mysqld.sock |
| sort_buffer_size | 2097144 |
| sql_mode | 0 |
| table_cache | 1800 |
| table_type | MYISAM |
| thread_cache_size | 384 |
| thread_stack | 196608 |
| tx_isolation | REPEATABLE-READ |
| timezone | CEST |
| tmp_table_size | 268435456 |
| tmpdir | /tmp/ |
| transaction_alloc_block_size | 8192 |
| transaction_prealloc_size | 4096 |
| version | 4.0.23_Debian-4-log |
| version_comment | Source distribution |
| version_compile_os | pc-linux-gnu |
| wait_timeout | 45 |
+---------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+--------------------------+------------+
| Variable_name | Value |
+--------------------------+------------+
| Aborted_clients | 2038 |
| Aborted_connects | 1 |
| Bytes_received | 3528583469 |
| Bytes_sent | 645665757 |
| Com_admin_commands | 4 |
| Com_alter_table | 5 |
| Com_analyze | 0 |
| Com_backup_table | 0 |
| Com_begin | 0 |
| Com_change_db | 760706 |
| Com_change_master | 0 |
| Com_check | 258 |
| Com_commit | 0 |
| Com_create_db | 0 |
| Com_create_function | 0 |
| Com_create_index | 0 |
| Com_create_table | 15 |
| Com_delete | 25084 |
| Com_delete_multi | 0 |
| Com_drop_db | 0 |
| Com_drop_function | 0 |
| Com_drop_index | 0 |
| Com_drop_table | 16 |
| Com_flush | 2 |
| Com_grant | 0 |
| Com_ha_close | 0 |
| Com_ha_open | 0 |
| Com_ha_read | 0 |
| Com_insert | 151252 |
| Com_insert_select | 2056 |
| Com_kill | 0 |
| Com_load | 0 |
| Com_load_master_data | 0 |
| Com_load_master_table | 0 |
| Com_lock_tables | 15 |
| Com_optimize | 1 |
| Com_purge | 0 |
| Com_rename_table | 0 |
| Com_repair | 0 |
| Com_replace | 0 |
| Com_replace_select | 0 |
| Com_reset | 0 |
| Com_restore_table | 0 |
| Com_revoke | 0 |
| Com_rollback | 0 |
| Com_savepoint | 0 |
| Com_select | 1511705 |
| Com_set_option | 451 |
| Com_show_binlog_events | 0 |
| Com_show_binlogs | 20 |
| Com_show_create | 457 |
| Com_show_databases | 74 |
| Com_show_fields | 494 |
| Com_show_grants | 0 |
| Com_show_keys | 24 |
| Com_show_logs | 0 |
| Com_show_master_status | 0 |
| Com_show_new_master | 0 |
| Com_show_open_tables | 0 |
| Com_show_processlist | 1 |
| Com_show_slave_hosts | 0 |
| Com_show_slave_status | 0 |
| Com_show_status | 2 |
| Com_show_innodb_status | 0 |
| Com_show_tables | 1346 |
| Com_show_variables | 6 |
| Com_slave_start | 0 |
| Com_slave_stop | 0 |
| Com_truncate | 0 |
| Com_unlock_tables | 245 |
| Com_update | 1638857 |
| Connections | 649385 |
| Created_tmp_disk_tables | 8997 |
| Created_tmp_tables | 312851 |
| Created_tmp_files | 255 |
| Delayed_insert_threads | 0 |
| Delayed_writes | 0 |
| Delayed_errors | 0 |
| Flush_commands | 1 |
| Handler_commit | 12 |
| Handler_delete | 5905 |
| Handler_read_first | 146834 |
| Handler_read_key | 3907854450 |
| Handler_read_next | 2563425473 |
| Handler_read_prev | 24805688 |
| Handler_read_rnd | 110514110 |
| Handler_read_rnd_next | 3354844906 |
| Handler_rollback | 177169 |
| Handler_update | 1018374211 |
| Handler_write | 1064087654 |
| Key_blocks_used | 124690 |
| Key_read_requests | 145083098 |
| Key_reads | 344489 |
| Key_write_requests | 155527 |
| Key_writes | 113 |
| Max_used_connections | 56 |
| Not_flushed_key_blocks | 19841 |
| Not_flushed_delayed_rows | 0 |
| Open_tables | 647 |
| Open_files | 625 |
| Open_streams | 0 |
| Opened_tables | 917 |
| Questions | 16448226 |
| Qcache_queries_in_cache | 26751 |
| Qcache_inserts | 1326541 |
| Qcache_hits | 11706754 |
| Qcache_lowmem_prunes | 0 |
| Qcache_not_cached | 185164 |
| Qcache_free_memory | 135797576 |
| Qcache_free_blocks | 11978 |
| Qcache_total_blocks | 65608 |
| Rpl_status | NULL |
| Select_full_join | 14429 |
| Select_full_range_join | 0 |
| Select_range | 12550 |
| Select_range_check | 0 |
| Select_scan | 417073 |
| Slave_open_temp_tables | 0 |
| Slave_running | OFF |
| Slow_launch_threads | 0 |
| Slow_queries | 1771 |
| Sort_merge_passes | 0 |
| Sort_range | 87330 |
| Sort_rows | 156415698 |
| Sort_scan | 331071 |
| Table_locks_immediate | 4358543 |
| Table_locks_waited | 2174 |
| Threads_cached | 45 |
| Threads_created | 57 |
| Threads_connected | 12 |
| Threads_running | 5 |
| Uptime | 110026 |
+--------------------------+------------+
Uptime: 110026 Threads: 12 Questions: 16448227 Slow queries: 1771 Opens: 917 Flush tables: 1 Open tables: 647 Queries per second avg: 149.494
mysqladmin Ver 8.40 Distrib 4.0.23, for pc-linux-gnu on i386
Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
This software comes with ABSOLUTELY NO WARRANTY. This is free software,
and you are welcome to modify and redistribute it under the GPL license

Server version 4.0.23_Debian-4-log
Protocol version 10
Connection Localhost via UNIX socket
UNIX socket /var/run/mysqld/mysqld.sock
Uptime: 1 day 6 hours 33 min 46 sec

Threads: 12 Questions: 16448227 Slow queries: 1771 Opens: 917 Flush tables: 1 Open tables: 647 Queries per second avg: 149.494

8.- I use Postnuke .726 and vBulletin

9.- Concurrent users: I have an average about 250 users every 15 minutes, with maximums of 350 or so. Cookie timeout: 7200 seconds.

10.- http://www.hispasonic.com/phpinfo.php

11.-
KeepAlive Off
MaxKeepAliveRequests 150
KeepAliveTimeout 0
MinSpareServers 15
MaxSpareServers 20
StartServers 15
MaxClients 180

12.- vB 3.5.4

13.- server:~# find / -size +2000000k
/var/lib/mysql/ibdata1
/var/lib/mysql/ibdata10BORRAR
/var/log/apache/access.log.1


14.-server:~# uname -a
Linux server.hispasonic.com 2.4.25-bf2.4 #1 SMP Wed Mar 10 10:35:09 PST 2004 i686 GNU/Linux
server:~# ulimit -aH
core file size (blocks, -c) unlimited
data seg size (kbytes, -d) unlimited
file size (blocks, -f) unlimited
max locked memory (kbytes, -l) unlimited
max memory size (kbytes, -m) unlimited
open files (-n) 1024
pipe size (512 bytes, -p) 8
stack size (kbytes, -s) unlimited
cpu time (seconds, -t) unlimited
max user processes (-u) unlimited
virtual memory (kbytes, -v) unlimited
server:~# cat /proc/cpuinfo
processor : 0
vendor_id : GenuineIntel
cpu family : 15
model : 2
model name : Intel(R) Xeon(TM) CPU 2.80GHz
stepping : 9
cpu MHz : 2790.766
cache size : 512 KB
fdiv_bug : no
hlt_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 2
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe cid
bogomips : 5570.56

processor : 1
vendor_id : GenuineIntel
cpu family : 15
model : 2
model name : Intel(R) Xeon(TM) CPU 2.80GHz
stepping : 9
cpu MHz : 2790.766
cache size : 512 KB
fdiv_bug : no
hlt_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 2
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe cid
bogomips : 5570.56

processor : 2
vendor_id : GenuineIntel
cpu family : 15
model : 2
model name : Intel(R) Xeon(TM) CPU 2.80GHz
stepping : 9
cpu MHz : 2790.766
cache size : 512 KB
fdiv_bug : no
hlt_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 2
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe cid
bogomips : 5570.56

processor : 3
vendor_id : GenuineIntel
cpu family : 15
model : 2
model name : Intel(R) Xeon(TM) CPU 2.80GHz
stepping : 9
cpu MHz : 2790.766
cache size : 512 KB
fdiv_bug : no
hlt_bug : no
f00f_bug : no
coma_bug : no
fpu : yes
fpu_exception : yes
cpuid level : 2
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe cid
bogomips : 5570.56

eva2000
Wed 26th Apr '06, 10:44am
what are the exact sizes of the files >2GB in values ?

1. Upgrade MySQL server to 4.0.26
2. Upgrade PHP to 4.4.2 if you use phpadsnew make sure to update to phpadsnew 2.0.8 or higher to work with PHP 4.4.1/4.4.2
3. Edit /etc/my.cnf and place the following mysql server settings in /etc/my.cnf and restart mysql server afterwards


[client]
port = 3306
socket = /var/run/mysqld/mysqld.sock

[mysqld]
skip-networking
language = /usr/share/mysql/spanish
port = 3306
socket = /var/run/mysqld/mysqld.sock
safe-show-database
max_connections = 650
key_buffer = 192M
myisam_sort_buffer_size = 64M
join_buffer_size = 1M
read_buffer_size = 1M
sort_buffer_size = 2M
table_cache = 1800
thread_cache_size = 384
wait_timeout = 30
connect_timeout = 10
tmp_table_size = 320M
max_heap_table_size = 64M
max_allowed_packet = 64M
max_connect_errors = 10
read_rnd_buffer_size = 524288
bulk_insert_buffer_size = 8M
query_cache_limit = 6M
query_cache_size = 128M
query_cache_type = 1
query_prealloc_size = 163840
query_alloc_block_size = 32768

# innodb settings
innodb_data_home_dir = /var/lib/mysql/
innodb_data_file_path = ibdata1:10M:autoextend
innodb_log_group_home_dir = /var/lib/mysql/
innodb_log_arch_dir = /var/lib/mysql/
innodb_buffer_pool_size=384M
innodb_additional_mem_pool_size=64M
innodb_log_file_size=64M
innodb_log_buffer_size=8M
innodb_flush_log_at_trx_commit=1
innodb_open_files = 300
innodb_status_file
innodb_thread_concurrency = 8

[mysqld_safe]
open_files_limit = 8192

[mysqldump]
quick
max_allowed_packet = 16M

[myisamchk]
key_buffer = 64M
sort_buffer = 64M
read_buffer = 16M
write_buffer = 16M

[mysqlhotcopy]
interactive-timeout


4. Install APC Cache lastest version from http://pecl.php.net/package/APC. Read install guide at http://www.vbulletin.com/forum/showthread.php?t=165367 - please remove Zend Optimizer from php.ini before installing APC Cache

5. If you just upgraded to vB 3.5.x try to disable these 4 options:

Admin CP -> vBulletin Options -> Forums Home Page Options -> Display Logged in Users?

Admin CP -> vBulletin Options -> Forum Display Options (forumdisplay) -> Show Users Browsing Forums

Admin CP -> vBulletin Options -> Thread Display Options -> Show Users Browsing Thread

Admin CP -> vBulletin Options -> Message Searching Options -> Automatic Similar Thread search

Soyuz
Wed 26th Apr '06, 11:06am
This does not work...

server:~# /etc/init.d/mysql restart
Stopping MySQL database server: mysqld.
Starting MySQL database server: mysqld...failed.
Please take a look at the syslog.

(I don't know where is the syslog...)

File sizes as you requested:

/var/lib/mysql/ibdata1 - 2.02 Gb
/var/lib/mysql/ibdata10BORRAR - 2.67 Gb
/var/log/apache/access.log.1 - 2.43 Gb

eva2000
Wed 26th Apr '06, 11:17am
take a look in yourhostname.err mysql error log in mysql data partition /var/lib/mysql to see what the error message is i.e.

tail -500 /var/lib/mysql/yourhostname.err

in ssh telnet as root user will show last 500 entries in the log

Soyuz
Wed 26th Apr '06, 1:07pm
I only see a big list of "ib_arch_log_0000000000" kind of files, some directories, ibdata & ib_log files, and a server.hispasonic.com.pid file in /var/lib/mysql/

I don't see any .err file... Thank you.

eva2000
Thu 27th Apr '06, 10:39am
try typing these 2 commands to find the err file

updatedb

locate *.err

Soyuz
Thu 27th Apr '06, 12:58pm
I located the file under /var/log/. But it's empty. I see mysql.err, mysql.err.1.gz, etc... empty or 20 bytes files.

:-(

eva2000
Fri 28th Apr '06, 3:51am
If you want submit a support ticket provide server ssh telnet details at http://www.vbulletin.com/members/membersupport_contactform.php to my attention and i can take a look into your server to see where that pesky error log is :)

it could be you have mysql.err rotated daily and mysql.err.1.gz is the rotated error log which you will need to unzip to read (take note of date stamps of the files)