Data migration( cpanel)

TRANSFER
—————
#!/bin/bash

for i in `cat /etc/trueuserdomains | awk -F: ‘{print $2}’`
do
/bin/echo “$i”
/scripts/pkgacct $i
file=$(ls /home/cpmove*$i*)
/bin/echo “$file”
scp -P -B $file root@:/root
wait
ls -lh $file
rm -rf $file
done

RESTORE
————–
ls cpmove-*.tar.gz | awk -F- ‘{print $2}’ | awk -F. ‘{print $1}’ > file
for i in `cat file`; do /scripts/restorepkg $i; done

First Install nginx,varnish and apache.

Install nginx
*********************
Type the following commands to download nginx, enter:
# cd /opt
# wget http://sysoev.ru/nginx/nginx-0.8.33.tar.gz

Untar nginx, enter:
# tar -zxvf nginx-0.8.33.tar.gz
# cd nginx-0.8.33

Configure nginx for 64 bit RHEL / CentOS Linux:
# ./configure –without-http_autoindex_module –without-http_ssi_module –without-http_userid_module –without-http_auth_basic_module –without-http_geo_module –without-http_fastcgi_module –without-http_empty_gif_module –with-openssl=/lib64

# make
# make install
/usr/local/nginx/sbin/nginx
/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf

*********************************

Change the port of apache to 81 and nginx to 80 so that the webserver port is 80.
/usr/local/nginx/sbin/nginx -v
Default nginx.conf (/usr/local/nginx/conf/nginx.conf)

*********************************************************

#user nobody;
worker_processes 1;

#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;

#pid logs/nginx.pid;

events {
worker_connections 1024;
}

http {
include mime.types;
default_type application/octet-stream;

#log_format main ‘$remote_addr – $remote_user [$time_local] “$request” ‘
# ‘$status $body_bytes_sent “$http_referer” ‘
# ‘”$http_user_agent” “$http_x_forwarded_for”‘;

#access_log logs/access.log main;

sendfile on;
#tcp_nopush on;

#keepalive_timeout 0;
keepalive_timeout 65;
upstream cache_servers {
server localhost:6081 max_fails=3 fail_timeout=30s;
}
#gzip on;
server {
access_log off;

error_log logs/vhost-error_log warn;
listen 80;
server_name emegano.com;

# uncomment location below to make nginx serve static files instead of Apache
# NOTE this will cause issues with bandwidth accounting as files wont be logged
location ~* \.(gif|jpg|jpeg|3gp|wmv|avi|mpg|mpeg|mp4|js|css)$ {
root /var/www/html/emegano.com;
}

location / {
proxy_send_timeout 90;
proxy_read_timeout 90;

proxy_buffer_size 4k;
# you can increase proxy_buffers here to suppress “an upstream response
# is buffered to a temporary file” warning
proxy_buffers 16 32k;
proxy_busy_buffers_size 64k;
proxy_temp_file_write_size 64k;

proxy_connect_timeout 30s;

# proxy_redirect https://.emegano.com:81 https://.emegano.com;
# proxy_redirect http://emegano.com:81 http://emegano.com;

proxy_pass http://41.223.52.100:6081;
#proxy_pass http://cache_servers/;
# proxy_redirect http://cache_servers/ http://$host:$server_port/;

proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}

# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
# listen 8000;
# listen somename:8080;
# server_name somename alias another.alias;

# location / {
# root html;
# index index.html index.htm;
# }
#}

# HTTPS server
#
#server {
# listen 443;
# server_name localhost;

# ssl on;
# ssl_certificate cert.pem;
# ssl_certificate_key cert.key;

# ssl_session_timeout 5m;

# ssl_protocols SSLv2 SSLv3 TLSv1;
# ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP;
# ssl_prefer_server_ciphers on;

# location / {
# root html;
# index index.html index.htm;
# }
#}

}
*****************************************************
yum install logrotate libgomp gcc cpp binutils kernel-headers glibc-headers glibc-devel
rpm -Uvh varnish-1.1.2-5el5.x86_64.rpm varnish-libs-1.1.2-5el5.x86_64.rpm
yum install varnish.x86_64
for A in varnish varnishlog ; do chkconfig $A on ; service $A start ; done
Varnish default.vcl
*******************************************************
cat /etc/varnish/default.vcl
#This is a basic VCL configuration file for varnish. See the vcl(7)
#man page for details on VCL syntax and semantics.
#
#Default backend definition. Set this to point to your content
#server.

backend default {
.host = “41.223.52.100”;
.port = “81”;
}
#
#Below is a commented-out copy of the default VCL logic. If you
#redefine any of these subroutines, the built-in logic will be
#appended to your code.
#
#sub vcl_recv {
# if (req.request != “GET” &&
# req.request != “HEAD” &&
# req.request != “PUT” &&
# req.request != “POST” &&
# req.request != “TRACE” &&
#req.request != “OPTIONS” &&
# req.request != “DELETE”) {
# /* Non-RFC2616 or CONNECT which is weird. */
# return (pipe);
# }
# if (req.request != “GET” && req.request != “HEAD”) {
# /* We only deal with GET and HEAD by default */
# return (pass);
# }
# if (req.http.Authorization || req.http.Cookie) {
# /* Not cacheable by default */
# return (pass);
# }
# return (lookup);
#}
#
#sub vcl_pipe {
# # Note that only the first request to the backend will have
# # X-Forwarded-For set. If you use X-Forwarded-For and want to
# # have it set for all requests, make sure to have:
# # set req.http.connection = “close”;
# # here. It is not set by default as it might break some broken web
# # applications, like IIS with NTLM authentication.
# return (pipe);
#}
#
#sub vcl_pass {
# return (pass);
#}
#
#sub vcl_hash {
# set req.hash += req.url;
# if (req.http.host) {
# set req.hash += req.http.host;
# } else {
# set req.hash += server.ip;
# }
# return (hash);
#}
#
#sub vcl_hit {
# if (!obj.cacheable) {
# return (pass);
# }
# return (deliver);
#}
#
#sub vcl_miss {
# return (fetch);
#}
#
#sub vcl_fetch {
# if (!obj.cacheable) {
# return (pass);
# }
# if (obj.http.Set-Cookie) {
# return (pass);
# }
# set obj.prefetch = -30s;
# return (deliver);
#}
#
#sub vcl_deliver {
# return (deliver);
#}
#
#sub vcl_discard {
# /* XXX: Do not redefine vcl_discard{}, it is not yet supported */
# return (discard);
#}
#
#sub vcl_prefetch {
# /* XXX: Do not redefine vcl_prefetch{}, it is not yet supported */
# return (fetch);
#}
#
#sub vcl_timeout {
# /* XXX: Do not redefine vcl_timeout{}, it is not yet supported */
# return (discard);
#}
#
#sub vcl_error {
# set obj.http.Content-Type = “text/html; charset=utf-8″;
# synthetic {”

# “} obj.status ” ” obj.response {”
#
#
# Error “} obj.status ” ” obj.response {”
#

“} obj.response {“

# Guru Meditation:
# XID: “} req.xid {”
#

---

#

# Varnish cache server
#

#};
# return (deliver);
#}

***************************************************
Varnish
***************************************************
cat /etc/sysconfig/varnish
# Configuration file for varnish
#
# /etc/init.d/varnish expects the variable $DAEMON_OPTS to be set from this
# shell script fragment.
#

# Maximum number of open files (for ulimit -n)
NFILES=131072

# Locked shared memory (for ulimit -l)
# Default log size is 82MB + header
MEMLOCK=82000

# Maximum size of corefile (for ulimit -c). Default in Fedora is 0
# DAEMON_COREFILE_LIMIT=”unlimited”

# This file contains 4 alternatives, please use only one.

## Alternative 1, Minimal configuration, no VCL
#
# Listen on port 6081, administration on localhost:6082, and forward to
# content server on localhost:8080. Use a fixed-size cache file.
#
#DAEMON_OPTS=”-a 41.223.52.100:6081 \
# -T localhost:6082 \
# -b localhost:8080 \
# -u varnish -g varnish \
# -s file,/var/lib/varnish/varnish_storage.bin,1G”

## Alternative 2, Configuration with VCL
#
# Listen on port 6081, administration on localhost:6082, and forward to
# one content server selected by the vcl file, based on the request. Use a
# fixed-size cache file.
#
DAEMON_OPTS=”-a 41.223.52.100:6081 \
-T localhost:6082 \
-f /etc/varnish/default.vcl \
-u varnish -g varnish \
-s file,/var/lib/varnish/varnish_storage.bin,1G”

## Alternative 3, Advanced configuration
#
# See varnishd(1) for more information.
#
# # Main configuration file. You probably want to change it 🙂
VARNISH_VCL_CONF=/etc/varnish/default.vcl
#
# # Default address and port to bind to
# # Blank address means all IPv4 and IPv6 interfaces, otherwise specify
# # a host name, an IPv4 dotted quad, or an IPv6 address in brackets.
# VARNISH_LISTEN_ADDRESS=
VARNISH_LISTEN_PORT=6081
#
# # Telnet admin interface listen address and port
VARNISH_ADMIN_LISTEN_ADDRESS=127.0.0.1
VARNISH_ADMIN_LISTEN_PORT=6082
#
# # The minimum number of worker threads to start
VARNISH_MIN_THREADS=1
#
# # The Maximum number of worker threads to start
VARNISH_MAX_THREADS=1000
#
# # Idle timeout for worker threads
VARNISH_THREAD_TIMEOUT=120
#
# # Cache file location
VARNISH_STORAGE_FILE=/var/lib/varnish/varnish_storage.bin
#
# # Cache file size: in bytes, optionally using k / M / G / T suffix,
# # or in percentage of available disk space using the % suffix.
VARNISH_STORAGE_SIZE=1G
#
# # Backend storage specification
VARNISH_STORAGE=”file,${VARNISH_STORAGE_FILE},${VARNISH_STORAGE_SIZE}”
#
# # Default TTL used when the backend does not specify one
VARNISH_TTL=120
#
# # DAEMON_OPTS is used by the init script. If you add or remove options, make
# # sure you update this section, too.
DAEMON_OPTS=”-a ${VARNISH_LISTEN_ADDRESS}:${VARNISH_LISTEN_PORT} \
-f ${VARNISH_VCL_CONF} \
-T ${VARNISH_ADMIN_LISTEN_ADDRESS}:${VARNISH_ADMIN_LISTEN_PORT} \
-t ${VARNISH_TTL} \
-w ${VARNISH_MIN_THREADS},${VARNISH_MAX_THREADS},${VARNISH_THREAD_TIMEOUT} \
-u varnish -g varnish \
-s ${VARNISH_STORAGE}”

## Alternative 4, Do It Yourself. See varnishd(1) for more information.
#
# DAEMON_OPTS=””
*******************************************************************************************************************************************

The steps are given below:

cd /usr/src
wget http://sysoev.ru/nginx/nginx-0.7.63.tar.gz
wget http://stderr.net/apache/rpaf/download/mod_rpaf-0.6.tar.gz
wget ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-7.9.tar.gz
tar xvzf mod_rpaf-0.6.tar.gz
cd mod_rpaf-0.6
/usr/local/apache/bin/apxs -i -c -n mod_rpaf-2.0.so mod_rpaf-2.0.c
cd /usr/src
tar xvzf pcre-7.9.tar.gz
cd pcre-7.9
./configure
make
make install
cd /usr/src
tar xvzf nginx-0.7.63.tar.gz
cd nginx-0.7.63
./configure –with-http_ssl_module –with-http_realip_module –with-http_dav_module –with-http_flv_module –with-http_gzip_static_module
make
make install

-Login to WHM and look for the following route:
Main >> Service Configuration >> Apache Setup > Include Editor > Pre Main Include

– Add the following configuration and save, LIST_OF_YOUR_IPS changing the IP you occupy in your site:

LoadModule rpaf_module modules/mod_rpaf-2.0.so
RPAFenable On
# Enable reverse proxy add forward
RPAFproxy_ips 127.0.0.1 LIST_OF_YOUR_IPS
# which ips are forwarding requests to us
RPAFsethostname On
# let rpaf update vhost settings
# allows to have the same hostnames as in the “real”
# configuration for the forwarding Apache
RPAFheader X-Real-IP
# Allows you to change which header mod_rpaf looks
# for when trying to find the ip the that is forwarding
# our requests

Change the apache port to 81 (tweak settings)
Open whm – tweak settings and find 0.0.0.0:80 and change it to 0.0.0.0.:81

– Run SSH console

/usr/local/cpanel/whostmgr/bin/whostmgr2 –updatetweaksettings
/scripts/rebuildhttpdconf
httpd restart

wget the automated nginx virtual host creator and run it

http://prajizworld.com/nginx.sh

chmod 755 nginx.sh

./nginx.sh

Remove the 3 # signs of the following lines from

/usr/local/nginx/conf/vhost.conf

#location ~* \.(gif|jpg|jpeg|png|wmv|avi|mpg|mpeg|mp4|js|css)$ {

# root /home/myweb/public_html;

#}

Check nginx configuration using

/usr/local/nginx/sbin/nginx -t

If everything is fine, its time to start nginx.

/usr/local/nginx/sbin/nginx

Configure nginx startup script

http://prajizworld.com/nginx.txt

mv nginx.txt /etc/init.d/nginx

chmod +x nginx

chkconfig –add –level 3 nginx

After doing all these, the server load was very low and pages were processing at lightning fast,

but still we experienced a slowness.

When I analyzed the traffic, I understood now network is bottleneck.

I could see many concurrent connections from same session downloading huge files.

So I thought of adding the below lines to limit traffic from same session

### Directive describes the zone, in which the session states are stored i.e. store in slimits. ###
### 1m can handle 32000 sessions with 32 bytes/session, set to 5m x 32000 session ###
limit_zone slimits $binary_remote_addr 5m;

### Control maximum number of simultaneous connections for one session i.e. ###
### restricts the amount of connections from a single ip address ###
limit_conn slimits 2;